parsl 2024.4.22__tar.gz → 2024.4.29__tar.gz

{parsl-2024.4.22/parsl.egg-info → parsl-2024.4.29}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.4.22
+Version: 2024.4.29
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.04.22.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.29.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/config.py

@@ -10,11 +10,12 @@ from parsl.executors.threads import ThreadPoolExecutor
 from parsl.errors import ConfigurationError
 from parsl.dataflow.taskrecord import TaskRecord
 from parsl.monitoring import MonitoringHub
+from parsl.usage_tracking.api import UsageInformation
 
 logger = logging.getLogger(__name__)
 
 
-class Config(RepresentationMixin):
+class Config(RepresentationMixin, UsageInformation):
     """
     Specification of Parsl configuration options.
 
@@ -50,6 +51,9 @@ class Config(RepresentationMixin):
         of 1.
     run_dir : str, optional
         Path to run directory. Default is 'runinfo'.
+    std_autopath : function, optional
+        Sets the function used to generate stdout/stderr specifications when parsl.AUTO_LOGPATH is used. If no function
+        is specified, generates paths that look like: ``rundir/NNN/task_logs/X/task_{id}_{name}{label}.{out/err}``
     strategy : str, optional
         Strategy to use for scaling blocks according to workflow needs. Can be 'simple', 'htex_auto_scale', 'none'
         or `None`.
@@ -89,6 +93,7 @@ class Config(RepresentationMixin):
                  retries: int = 0,
                  retry_handler: Optional[Callable[[Exception, TaskRecord], float]] = None,
                  run_dir: str = 'runinfo',
+                 std_autopath: Optional[Callable] = None,
                  strategy: Optional[str] = 'simple',
                  strategy_period: Union[float, int] = 5,
                  max_idletime: float = 120.0,
@@ -129,6 +134,7 @@ class Config(RepresentationMixin):
         self.usage_tracking = usage_tracking
         self.initialize_logging = initialize_logging
         self.monitoring = monitoring
+        self.std_autopath: Optional[Callable] = std_autopath
 
     @property
     def executors(self) -> Sequence[ParslExecutor]:
@@ -144,3 +150,6 @@ class Config(RepresentationMixin):
         if len(duplicates) > 0:
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
+
+    def get_usage_information(self):
+        return {"executors_len": len(self.executors)}

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/data_provider/zip.py

@@ -42,6 +42,12 @@ class ZipFileStaging(Staging):
     """
 
     def can_stage_out(self, file: File) -> bool:
+        return self.is_zip_url(file)
+
+    def can_stage_in(self, file: File) -> bool:
+        return self.is_zip_url(file)
+
+    def is_zip_url(self, file: File) -> bool:
         logger.debug("archive provider checking File {}".format(repr(file)))
 
         # First check if this is the scheme we care about
@@ -76,6 +82,20 @@ class ZipFileStaging(Staging):
         app_fut = stage_out_app(zip_path, inside_path, working_dir, inputs=[file], _parsl_staging_inhibit=True, parent_fut=parent_fut)
         return app_fut
 
+    def stage_in(self, dm, executor, file, parent_fut):
+        assert file.scheme == 'zip'
+
+        zip_path, inside_path = zip_path_split(file.path)
+
+        working_dir = dm.dfk.executors[executor].working_dir
+
+        if working_dir:
+            file.local_path = os.path.join(working_dir, inside_path)
+
+        stage_in_app = _zip_stage_in_app(dm)
+        app_fut = stage_in_app(zip_path, inside_path, working_dir, outputs=[file], _parsl_staging_inhibit=True, parent_fut=parent_fut)
+        return app_fut._outputs[0]
+
 
 def _zip_stage_out(zip_file, inside_path, working_dir, parent_fut=None, inputs=[], _parsl_staging_inhibit=True):
     file = inputs[0]
@@ -93,6 +113,18 @@ def _zip_stage_out_app(dm):
     return parsl.python_app(executors=['_parsl_internal'], data_flow_kernel=dm.dfk)(_zip_stage_out)
 
 
+def _zip_stage_in(zip_file, inside_path, working_dir, *, parent_fut, outputs, _parsl_staging_inhibit=True):
+    with filelock.FileLock(zip_file + ".lock"):
+        with zipfile.ZipFile(zip_file, mode='r') as z:
+            content = z.read(inside_path)
+        with open(outputs[0], "wb") as of:
+            of.write(content)
+
+
+def _zip_stage_in_app(dm):
+    return parsl.python_app(executors=['_parsl_internal'], data_flow_kernel=dm.dfk)(_zip_stage_in)
+
+
 def zip_path_split(path: str) -> Tuple[str, str]:
     """Split zip: path into a zipfile name and a contained-file name.
     """

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/dataflow/dflow.py

@@ -798,7 +798,6 @@ class DataFlowKernel:
         # be the original function wrapped with an in-task stageout wrapper), a
         # rewritten File object to be passed to task to be executed
 
-        @typechecked
         def stageout_one_file(file: File, rewritable_func: Callable):
             if not self.check_staging_inhibited(kwargs):
                 # replace a File with a DataFuture - either completing when the stageout
@@ -996,32 +995,16 @@ class DataFlowKernel:
         executor = random.choice(choices)
         logger.debug("Task {} will be sent to executor {}".format(task_id, executor))
 
-        # The below uses func.__name__ before it has been wrapped by any staging code.
-
-        label = app_kwargs.get('label')
-        for kw in ['stdout', 'stderr']:
-            if kw in app_kwargs:
-                if app_kwargs[kw] == parsl.AUTO_LOGNAME:
-                    if kw not in ignore_for_cache:
-                        ignore_for_cache += [kw]
-                    app_kwargs[kw] = os.path.join(
-                                self.run_dir,
-                                'task_logs',
-                                str(int(task_id / 10000)).zfill(4),  # limit logs to 10k entries per directory
-                                'task_{}_{}{}.{}'.format(
-                                    str(task_id).zfill(4),
-                                    func.__name__,
-                                    '' if label is None else '_{}'.format(label),
-                                    kw)
-                    )
-
         resource_specification = app_kwargs.get('parsl_resource_specification', {})
 
         task_record: TaskRecord
-        task_record = {'depends': [],
+        task_record = {'args': app_args,
+                       'depends': [],
                        'dfk': self,
                        'executor': executor,
+                       'func': func,
                        'func_name': func.__name__,
+                       'kwargs': app_kwargs,
                        'memoize': cache,
                        'hashsum': None,
                        'exec_fu': None,
@@ -1043,18 +1026,30 @@ class DataFlowKernel:
 
         self.update_task_state(task_record, States.unsched)
 
+        for kw in ['stdout', 'stderr']:
+            if kw in app_kwargs:
+                if app_kwargs[kw] == parsl.AUTO_LOGNAME:
+                    if kw not in ignore_for_cache:
+                        ignore_for_cache += [kw]
+                    if self.config.std_autopath is None:
+                        app_kwargs[kw] = self.default_std_autopath(task_record, kw)
+                    else:
+                        app_kwargs[kw] = self.config.std_autopath(task_record, kw)
+
         app_fu = AppFuture(task_record)
+        task_record['app_fu'] = app_fu
 
         # Transform remote input files to data futures
         app_args, app_kwargs, func = self._add_input_deps(executor, app_args, app_kwargs, func)
 
         func = self._add_output_deps(executor, app_args, app_kwargs, app_fu, func)
 
+        # Replace the function invocation in the TaskRecord with whatever file-staging
+        # substitutions have been made.
         task_record.update({
                     'args': app_args,
                     'func': func,
-                    'kwargs': app_kwargs,
-                    'app_fu': app_fu})
+                    'kwargs': app_kwargs})
 
         assert task_id not in self.tasks
 
@@ -1245,8 +1240,10 @@ class DataFlowKernel:
                 self._checkpoint_timer.close()
 
         # Send final stats
+        logger.info("Sending end message for usage tracking")
         self.usage_tracker.send_end_message()
         self.usage_tracker.close()
+        logger.info("Closed usage tracking")
 
         logger.info("Closing job status poller")
         self.job_status_poller.close()
@@ -1440,6 +1437,19 @@ class DataFlowKernel:
         log_std_stream("Standard out", task_record['app_fu'].stdout)
         log_std_stream("Standard error", task_record['app_fu'].stderr)
 
+    def default_std_autopath(self, taskrecord, kw):
+        label = taskrecord['kwargs'].get('label')
+        task_id = taskrecord['id']
+        return os.path.join(
+            self.run_dir,
+            'task_logs',
+            str(int(task_id / 10000)).zfill(4),  # limit logs to 10k entries per directory
+            'task_{}_{}{}.{}'.format(
+                str(task_id).zfill(4),
+                taskrecord['func_name'],
+                '' if label is None else '_{}'.format(label),
+                kw))
+
 
 class DataFlowKernelLoader:
     """Manage which DataFlowKernel is active.

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/executors/high_throughput/executor.py

@@ -14,6 +14,7 @@ import math
 import warnings
 
 import parsl.launchers
+from parsl.usage_tracking.api import UsageInformation
 from parsl.serialize import pack_res_spec_apply_message, deserialize
 from parsl.serialize.errors import SerializationError, DeserializationError
 from parsl.app.errors import RemoteExceptionWrapper
@@ -62,7 +63,7 @@ DEFAULT_LAUNCH_CMD = ("process_worker_pool.py {debug} {max_workers_per_node} "
                       "--available-accelerators {accelerators}")
 
 
-class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
+class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageInformation):
     """Executor designed for cluster-scale
 
     The HighThroughputExecutor system has the following components:
@@ -818,4 +819,9 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
             logger.info("Unable to terminate Interchange process; sending SIGKILL")
             self.interchange_proc.kill()
 
+        self.interchange_proc.close()
+
         logger.info("Finished HighThroughputExecutor shutdown attempt")
+
+    def get_usage_information(self):
+        return {"mpi": self.enable_mpi_mode}

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/providers/kubernetes/kube.py

@@ -105,7 +105,26 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         if not _kubernetes_enabled:
             raise OptionalModuleMissing(['kubernetes'],
                                         "Kubernetes provider requires kubernetes module and config.")
-        config.load_kube_config()
+        try:
+            config.load_kube_config()
+        except config.config_exception.ConfigException:
+            # `load_kube_config` assumes a local kube-config file, and fails if not
+            # present, raising:
+            #
+            #     kubernetes.config.config_exception.ConfigException: Invalid
+            #     kube-config file. No configuration found.
+            #
+            # Since running a parsl driver script on a kubernetes cluster is a common
+            # pattern to enable worker-interchange communication, this enables an
+            # in-cluster config to be loaded if a kube-config file isn't found.
+            #
+            # Based on: https://github.com/kubernetes-client/python/issues/1005
+            try:
+                config.load_incluster_config()
+            except config.config_exception.ConfigException:
+                raise config.config_exception.ConfigException(
+                    "Failed to load both kube-config file and in-cluster configuration."
+                )
 
         self.namespace = namespace
         self.image = image

parsl-2024.4.29/parsl/tests/configs/local_threads_checkpoint_periodic.py

@@ -0,0 +1,11 @@
+from parsl.config import Config
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def fresh_config():
+    tpe = ThreadPoolExecutor(label='local_threads_checkpoint_periodic', max_threads=1)
+    return Config(
+        executors=[tpe],
+        checkpoint_mode='periodic',
+        checkpoint_period='00:00:02'
+    )

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/tests/conftest.py

@@ -3,8 +3,10 @@ import itertools
 import logging
 import os
 import pathlib
+import random
 import re
 import shutil
+import string
 import time
 import types
 import signal
@@ -139,7 +141,7 @@ def pytest_configure(config):
     )
     config.addinivalue_line(
         'markers',
-        'staging_required: Marks tests that require a staging provider, when there is no sharedFS)'
+        'staging_required: Marks tests that require a staging provider, when there is no sharedFS'
     )
     config.addinivalue_line(
         'markers',
@@ -245,6 +247,7 @@ def load_dfk_local_module(request, pytestconfig, tmpd_cwd_session):
 
         if callable(local_teardown):
             local_teardown()
+            assert DataFlowKernelLoader._dfk is None, "Expected teardown to clear DFK"
 
         if local_config:
             if parsl.dfk() != dfk:
@@ -421,3 +424,11 @@ def try_assert():
             raise AssertionError("Bad assert call: no attempts or timeout period")
 
     yield _impl
+
+
+@pytest.fixture
+def randomstring():
+    def func(length=5, alphabet=string.ascii_letters):
+        return "".join(random.choice(alphabet) for _ in range(length))
+
+    return func

parsl-2024.4.29/parsl/tests/test_bash_apps/test_std_uri.py

@@ -0,0 +1,128 @@
+import logging
+import parsl
+import pytest
+import zipfile
+
+from functools import partial
+from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
+from parsl.executors import ThreadPoolExecutor
+
+
+@parsl.bash_app
+def app_stdout(stdout=parsl.AUTO_LOGNAME):
+    return "echo hello"
+
+
+def const_str(cpath, task_record, err_or_out):
+    return cpath
+
+
+def const_with_cpath(autopath_specifier, content_path, caplog):
+    with parsl.load(parsl.Config(std_autopath=partial(const_str, autopath_specifier))):
+        fut = app_stdout()
+
+        # we don't have to wait for a result to check this attributes
+        assert fut.stdout is autopath_specifier
+
+        # there is no DataFuture to wait for in the str case: the model is that
+        # the stdout will be immediately available on task completion.
+        fut.result()
+
+    with open(content_path, "r") as file:
+        assert file.readlines() == ["hello\n"]
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+    parsl.clear()
+
+
+@pytest.mark.local
+def test_std_autopath_const_str(caplog, tmpd_cwd):
+    """Tests str and tuple mode autopaths with constant autopath, which should
+    all be passed through unmodified.
+    """
+    cpath = str(tmpd_cwd / "CONST")
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_pathlike(caplog, tmpd_cwd):
+    cpath = tmpd_cwd / "CONST"
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_tuples(caplog, tmpd_cwd):
+    file = tmpd_cwd / "CONST"
+    cpath = (file, "w")
+    const_with_cpath(cpath, file, caplog)
+
+
+class URIFailError(Exception):
+    pass
+
+
+def fail_uri(task_record, err_or_out):
+    raise URIFailError("Deliberate failure in std stream filename generation")
+
+
+@pytest.mark.local
+def test_std_autopath_fail(caplog):
+    with parsl.load(parsl.Config(std_autopath=fail_uri)):
+        with pytest.raises(URIFailError):
+            app_stdout()
+
+    parsl.clear()
+
+
+@parsl.bash_app
+def app_both(stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello; echo goodbye >&2"
+
+
+def zip_uri(base, task_record, err_or_out):
+    """Should generate Files in base.zip like app_both.0.out or app_both.123.err"""
+    zip_path = base / "base.zip"
+    file = f"{task_record['func_name']}.{task_record['id']}.{task_record['try_id']}.{err_or_out}"
+    return File(f"zip:{zip_path}/{file}")
+
+
+@pytest.mark.local
+def test_std_autopath_zip(caplog, tmpd_cwd):
+    with parsl.load(parsl.Config(run_dir=str(tmpd_cwd),
+                                 executors=[ThreadPoolExecutor(working_dir=str(tmpd_cwd))],
+                                 std_autopath=partial(zip_uri, tmpd_cwd))):
+        futs = []
+
+        for _ in range(10):
+            fut = app_both()
+
+            # assertions that should hold after submission
+            assert isinstance(fut.stdout, DataFuture)
+            assert fut.stdout.file_obj.url.startswith("zip")
+
+            futs.append(fut)
+
+        # Barrier for all the stageouts to complete so that we can
+        # poke at the zip file.
+        [(fut.stdout.result(), fut.stderr.result()) for fut in futs]
+
+        with zipfile.ZipFile(tmpd_cwd / "base.zip") as z:
+            for fut in futs:
+
+                assert fut.done(), "AppFuture should be done if stageout is done"
+
+                stdout_relative_path = f"app_both.{fut.tid}.0.stdout"
+                with z.open(stdout_relative_path) as f:
+                    assert f.readlines() == [b'hello\n']
+
+                stderr_relative_path = f"app_both.{fut.tid}.0.stderr"
+                with z.open(stderr_relative_path) as f:
+                    assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+    parsl.clear()

parsl-2024.4.29/parsl/tests/test_checkpointing/test_periodic.py

@@ -0,0 +1,53 @@
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.local_threads_checkpoint_periodic import fresh_config
+
+
+def local_setup():
+    parsl.load(fresh_config())
+
+
+def local_teardown():
+    # explicit clear without dfk.cleanup here, because the
+    # test does that already
+    parsl.clear()
+
+
+@python_app(cache=True)
+def slow_double(x, sleep_dur=1):
+    import time
+    time.sleep(sleep_dur)
+    return x * 2
+
+
+def tstamp_to_seconds(line):
+    f = line.partition(" ")[0]
+    return float(f)
+
+
+@pytest.mark.local
+def test_periodic():
+    """Test checkpointing with task_periodic behavior
+    """
+    h, m, s = map(int, parsl.dfk().config.checkpoint_period.split(":"))
+    assert h == 0, "Verify test setup"
+    assert m == 0, "Verify test setup"
+    assert s > 0, "Verify test setup"
+    sleep_for = s + 1
+    with parsl.dfk():
+        futs = [slow_double(sleep_for) for _ in range(4)]
+        [f.result() for f in futs]
+
+    # Here we will check if the loglines came back with 5 seconds deltas
+    with open("{}/parsl.log".format(parsl.dfk().run_dir)) as f:
+        log_lines = f.readlines()
+    expected_msg = " Done checkpointing"
+    expected_msg2 = " No tasks checkpointed in this pass"
+
+    lines = [line for line in log_lines if expected_msg in line or expected_msg2 in line]
+    assert len(lines) >= 3, "Insufficient checkpoint lines in logfile"
+    deltas = [tstamp_to_seconds(line) for line in lines]
+    assert deltas[1] - deltas[0] < 5.5, "Delta between checkpoints exceeded period"
+    assert deltas[2] - deltas[1] < 5.5, "Delta between checkpoints exceeded period"

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/tests/test_htex/test_basic.py

@@ -23,6 +23,6 @@ def dummy():
 
 
 @pytest.mark.local
-def test_that_it_fails():
+def test_app():
     x = dummy()
-    x.result()
+    assert x.result() is None

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/tests/test_htex/test_missing_worker.py

@@ -37,7 +37,3 @@ def test_that_it_fails():
         raise Exception("The app somehow ran without a valid worker")
 
     assert parsl.dfk().config.executors[0]._executor_bad_state.is_set()
-
-    # htex needs shutting down explicitly because dfk.cleanup() will not
-    # do that, as it is in bad state
-    parsl.dfk().config.executors[0].shutdown()

{parsl-2024.4.22 → parsl-2024.4.29}/parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -25,17 +25,11 @@ from parsl.executors.high_throughput.mpi_prefix_composer import (
 EXECUTOR_LABEL = "MPI_TEST"
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].label = EXECUTOR_LABEL
     config.executors[0].max_workers_per_node = 1
-    parsl.load(config)
-
-
-def local_teardown():
-    logging.warning("Exiting")
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl-2024.4.29/parsl/tests/test_staging/test_zip_in.py

@@ -0,0 +1,42 @@
+import parsl
+import pytest
+import random
+import zipfile
+
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_in(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    # create a zip file containing one file with some abitrary number of lines
+    n_lines = random.randint(0, 1000)
+
+    with zipfile.ZipFile(zip_path, mode='w') as z:
+        with z.open(file_base, mode='w') as f:
+            for _ in range(n_lines):
+                f.write(b'someline\n')
+
+    app_future = count_lines(zip_file)
+
+    assert app_future.result() == n_lines

parsl-2024.4.29/parsl/tests/test_staging/test_zip_to_zip.py

@@ -0,0 +1,44 @@
+import parsl
+import pytest
+import random
+import zipfile
+
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def generate_lines(n: int, *, outputs):
+    with open(outputs[0], "w") as f:
+        for x in range(n):
+            # write numbered lines
+            f.write(str(x) + "\n")
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_pipeline(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    n_lines = random.randint(0, 1000)
+    generate_fut = generate_lines(n_lines, outputs=[zip_file])
+    n_lines_out = count_lines(generate_fut.outputs[0]).result()
+
+    assert n_lines == n_lines_out