parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_staging/test_staging_stdout.py

@@ -0,0 +1,64 @@
+import logging
+import os
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.bash_app
+def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello ; echo goodbye >&2"
+
+
+@pytest.mark.staging_required
+def test_stdout_staging_file(tmpd_cwd, caplog):
+    basename = str(tmpd_cwd) + "/stdout.txt"
+    stdout_file = File("file://" + basename)
+
+    app_future = output_to_stds(stdout=stdout_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    assert os.path.exists(basename)
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+
+@pytest.mark.staging_required
+def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
+    zipfile_name = str(tmpd_cwd) + "/staging.zip"
+    stdout_relative_path = "somewhere/test-out.txt"
+    stdout_file = File("zip:" + zipfile_name + "/" + stdout_relative_path)
+
+    stderr_relative_path = "somewhere/test-error.txt"
+    stderr_file = File("zip:" + zipfile_name + "/" + stderr_relative_path)
+
+    app_future = output_to_stds(stdout=stdout_file, stderr=stderr_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    # check the file exists as soon as possible
+    assert os.path.exists(zipfile_name)
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stdout_relative_path) as f:
+            assert f.readlines() == [b'hello\n']
+
+    assert isinstance(app_future.stderr, DataFuture)
+    app_future.stderr.result()
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stderr_relative_path) as f:
+            # The last line of stderr should be goodbye, but Parsl will write
+            # other Parsl-specific into to stderr before that, so only assert
+            # the behaviour of the final line.
+            assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_staging/test_zip_in.py

@@ -0,0 +1,39 @@
+import random
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_in(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    # create a zip file containing one file with some abitrary number of lines
+    n_lines = random.randint(0, 1000)
+
+    with zipfile.ZipFile(zip_path, mode='w') as z:
+        with z.open(file_base, mode='w') as f:
+            for _ in range(n_lines):
+                f.write(b'someline\n')
+
+    app_future = count_lines(zip_file)
+
+    assert app_future.result() == n_lines

parsl/tests/test_staging/test_zip_out.py

@@ -0,0 +1,110 @@
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@pytest.mark.local
+def test_zip_path_split():
+    from parsl.data_provider.zip import zip_path_split
+    assert zip_path_split("/tmp/foo/this.zip/inside/here.txt") == ("/tmp/foo/this.zip", "inside/here.txt")
+
+
+@parsl.bash_app
+def output_something(outputs=()):
+    """This should output something into every specified output file:
+    the position in the output sequence will be written into the
+    corresponding output file.
+    """
+    cmds = []
+    for n in range(len(outputs)):
+        cmds.append(f"echo {n} > {outputs[n]}")
+
+    return "; ".join(cmds)
+
+
+@pytest.mark.local
+def test_zip_out(tmpd_cwd):
+    # basic test of zip file stage-out
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip:{zip_path / file_base}")
+
+    app_future = output_something(outputs=[of])
+    output_file_future = app_future.outputs[0]
+
+    app_future.result()
+    output_file_future.result()
+
+    assert zipfile.is_zipfile(zip_path)
+
+    with zipfile.ZipFile(zip_path) as z:
+        assert file_base in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(file_base) as f:
+            assert f.readlines() == [b'0\n']
+
+
+@pytest.mark.local
+def test_zip_out_multi(tmpd_cwd):
+    # tests multiple files, multiple zip files and multiple
+    # sub-paths
+
+    zip_path_1 = tmpd_cwd / "container1.zip"
+    zip_path_2 = tmpd_cwd / "container2.zip"
+
+    relative_file_path_1 = "a/b/c/data.txt"
+    relative_file_path_2 = "something.txt"
+    relative_file_path_3 = "a/d/other.txt"
+    of1 = File(f"zip:{zip_path_1 / relative_file_path_1}")
+    of2 = File(f"zip:{zip_path_1 / relative_file_path_2}")
+    of3 = File(f"zip:{zip_path_2 / relative_file_path_3}")
+
+    app_future = output_something(outputs=[of1, of2, of3])
+
+    for f in app_future.outputs:
+        f.result()
+
+    app_future.result()
+
+    assert zipfile.is_zipfile(zip_path_1)
+
+    with zipfile.ZipFile(zip_path_1) as z:
+        assert relative_file_path_1 in z.namelist()
+        assert relative_file_path_2 in z.namelist()
+        assert len(z.namelist()) == 2
+        with z.open(relative_file_path_1) as f:
+            assert f.readlines() == [b'0\n']
+        with z.open(relative_file_path_2) as f:
+            assert f.readlines() == [b'1\n']
+
+    assert zipfile.is_zipfile(zip_path_2)
+
+    with zipfile.ZipFile(zip_path_2) as z:
+        assert relative_file_path_3 in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(relative_file_path_3) as f:
+            assert f.readlines() == [b'2\n']
+
+
+@pytest.mark.local
+def test_zip_bad_authority(tmpd_cwd):
+    # tests that there's an exception when staging a ZIP url with an authority
+    # section specified, rather than silently ignoring it. This simulates a
+    # user who misunderstands what that piece of what a zip: URL means.
+
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip://someauthority/{zip_path / file_base}")
+
+    with pytest.raises(ZipAuthorityError):
+        output_something(outputs=[of])

parsl/tests/test_staging/test_zip_to_zip.py

@@ -0,0 +1,41 @@
+import random
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def generate_lines(n: int, *, outputs):
+    with open(outputs[0], "w") as f:
+        for x in range(n):
+            # write numbered lines
+            f.write(str(x) + "\n")
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_pipeline(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    n_lines = random.randint(0, 1000)
+    generate_fut = generate_lines(n_lines, outputs=[zip_file])
+    n_lines_out = count_lines(generate_fut.outputs[0]).result()
+
+    assert n_lines == n_lines_out

parsl/tests/test_summary.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl.tests.configs.local_threads import fresh_config
 
 
@@ -22,7 +23,6 @@ def test_summary(caplog):
     fail().exception()
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     assert "Summary of tasks in DFK:" in caplog.text
     assert "Tasks in state States.exec_done: 1" in caplog.text

parsl/tests/test_thread_parallelism.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_threads/test_configs.py

@@ -7,9 +7,9 @@ from parsl.tests.configs.local_threads import fresh_config
 
 @python_app
 def worker_identify(x, sleep_dur=0.2):
-    import time
     import os
     import threading
+    import time
     time.sleep(sleep_dur)
     return {"pid": os.getpid(),
             "tid": threading.current_thread()}
@@ -30,5 +30,4 @@ def test_parallel_for():
     assert thread_count <= config.executors[0].max_threads, "More threads than allowed"
     assert process_count == 1, "More processes than allowed"
     dfk.cleanup()
-    parsl.clear()
     return d

parsl/tests/test_threads/test_lazy_errors.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl import python_app
 from parsl.tests.configs.local_threads import fresh_config
 
@@ -24,5 +25,4 @@ def test_lazy_behavior():
         assert f.done()
 
     parsl.dfk().cleanup()
-    parsl.clear()
     return

parsl/tests/test_utils/test_execute_wait.py

@@ -0,0 +1,35 @@
+import pytest
+
+from parsl.utils import execute_wait
+
+
+@pytest.mark.local
+def test_env():
+    ''' Regression testing for issue #27
+    '''
+
+    rc, stdout, stderr = execute_wait("env", 1)
+
+    stdout = stdout.split('\n')
+    x = [s for s in stdout if s.startswith("PATH=")]
+    assert x, "PATH not found"
+
+    x = [s for s in stdout if s.startswith("HOME=")]
+    assert x, "HOME not found"
+
+
+@pytest.mark.local
+def test_large_output_2210():
+    """Regression test for #2210.
+    execute_wait was hanging if the specified command gave too
+    much output, due to a race condition between process exiting and
+    pipes filling up.
+    """
+
+    # this will output 128kb of stdout
+    execute_wait("yes | dd count=128 bs=1024", walltime=60)
+
+    # if this test fails, execute_wait should raise a timeout
+    # exception.
+
+    # The contents out the output is not verified by this test

parsl/tests/test_utils/test_sanitize_dns.py

@@ -0,0 +1,76 @@
+import random
+import re
+
+import pytest
+
+from parsl.utils import sanitize_dns_label_rfc1123, sanitize_dns_subdomain_rfc1123
+
+# Ref: https://datatracker.ietf.org/doc/html/rfc1123
+DNS_LABEL_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$'
+DNS_SUBDOMAIN_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$'
+
+test_labels = [
+    "example-label-123",           # Valid label
+    "EXAMPLE",                     # Case sensitivity
+    "!@#example*",                 # Remove invalid characters
+    "--leading-and-trailing--",    # Leading and trailing hyphens
+    "..leading.and.trailing..",    # Leading and tailing dots
+    "multiple..dots",              # Consecutive dots
+    "valid--label",                # Consecutive hyphens
+    "a" * random.randint(64, 70),  # Longer than 63 characters
+    f"{'a' * 62}-a",               # Trailing hyphen at max length
+]
+
+
+def _generate_test_subdomains(num_subdomains: int):
+    subdomains = []
+    for _ in range(num_subdomains):
+        num_labels = random.randint(1, 5)
+        labels = [test_labels[random.randint(0, num_labels - 1)] for _ in range(num_labels)]
+        subdomain = ".".join(labels)
+        subdomains.append(subdomain)
+    return subdomains
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", test_labels)
+def test_sanitize_dns_label_rfc1123(raw_string: str):
+    print(sanitize_dns_label_rfc1123(raw_string))
+    assert re.match(DNS_LABEL_REGEX, sanitize_dns_label_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", "-", "@", "$$$"))
+def test_sanitize_dns_label_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_label_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS label is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", _generate_test_subdomains(10))
+def test_sanitize_dns_subdomain_rfc1123(raw_string: str):
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("char", ("-", "."))
+def test_sanitize_dns_subdomain_rfc1123_trailing_non_alphanumeric_at_max_length(char: str):
+    raw_string = (f"{'a' * 61}." * 4) + f".aaaa{char}a"
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", ".", "..."))
+def test_sanitize_dns_subdomain_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_subdomain_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS subdomain is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "raw_string", ("a" * 253, "a" * random.randint(254, 300)), ids=("254 chars", ">253 chars")
+)
+def test_sanitize_dns_subdomain_rfc1123_max_length(raw_string: str):
+    assert len(sanitize_dns_subdomain_rfc1123(raw_string)) <= 253

parsl/tests/unit/test_address.py

@@ -0,0 +1,20 @@
+import pytest
+
+from parsl.addresses import tcp_url
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("address, port,expected", [
+    ("127.0.0.1", 55001, "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", "55001", "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", None, "tcp://127.0.0.1"),
+    ("::1", "55001", "tcp://[::1]:55001"),
+    ("::ffff:127.0.0.1", 55001, "tcp://[::ffff:127.0.0.1]:55001"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("*", None, "tcp://*"),
+])
+def test_tcp_url(address, port, expected):
+    """Confirm valid address generation"""
+    result = tcp_url(address, port)
+    assert result == expected

parsl/tests/unit/test_file.py

@@ -0,0 +1,99 @@
+import os
+from unittest import mock
+
+import pytest
+
+from parsl import File
+
+_MOCK_BASE = "parsl.data_provider.files."
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("http", "https", "ftp", "ftps", "asdfasdf"))
+def test_file_init_scheme(scheme):
+    basename = "some_base_name"
+    path = f"/some/path/1/2/3/{basename}"
+    fqdn = "some.fqdn.example.com"
+    exp_url = f"{scheme}://{fqdn}{path}"
+    f = File(exp_url)
+    assert f.url == exp_url, "Expected given url to be stored"
+    assert f.scheme == scheme
+    assert f.netloc == fqdn
+    assert f.path == path
+    assert f.filename == basename
+    assert f.local_path is None, "Expect only set by API consumer, not constructor"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("url", ("some weird :// url", "", "a"))
+def test_file_init_file_url_fallback(url):
+    exp_url = "some weird :// url"
+    f = File(exp_url)
+    assert f.url == exp_url
+    assert not f.netloc, "invalid host, should be no netloc"
+    assert f.path == exp_url, "Should fail to fully parse, so path is whole url"
+    assert f.filename == exp_url.rsplit("/", 1)[-1]
+
+    assert f.scheme == "file"
+
+
+@pytest.mark.local
+def test_file_proxies_for_filepath(randomstring):
+    # verify (current) expected internal hookup
+    exp_filepath = randomstring()
+    with mock.patch(
+        f"{_MOCK_BASE}File.filepath", new_callable=mock.PropertyMock
+    ) as mock_fpath:
+        mock_fpath.return_value = exp_filepath
+        f = File("")
+        assert str(f) == exp_filepath
+        assert os.fspath(f) == exp_filepath
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("file://", ""))
+def test_file_filepath_local_path_is_priority(scheme, randomstring):
+    exp_path = "/some/local/path"
+    url = f"{scheme}{exp_path}"
+    f = File(url)
+
+    f.local_path = randomstring()
+    assert f.filepath == f.local_path
+
+    f.local_path = None
+    assert f.filepath == exp_path
+
+
+@pytest.mark.local
+def test_file_filepath_requires_local_accessible_path():
+    with pytest.raises(ValueError) as pyt_exc:
+        _ = File("http://").filepath
+
+    assert "No local_path" in str(pyt_exc.value), "Expected reason in exception"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("https", "ftps", "", "file", "asdfasdf"))
+def test_file_repr(scheme):
+    netloc = "some.netloc"
+    filename = "some_file_name"
+    path = f"/some/path/{filename}"
+    if scheme:
+        url = f"{scheme}://{netloc}{path}"
+    else:
+        scheme = "file"
+        url = path
+
+    f = File(url)
+    r = repr(f)
+    assert r.startswith("<")
+    assert r.endswith(">")
+    assert f"<{type(f).__name__} " in r
+    assert f" at 0x{id(f):x}" in r
+    assert f" url={url}" in r
+    assert f" scheme={scheme}" in r
+    assert f" path={path}" in r
+    assert f" filename={filename}" in r
+
+    if scheme != "file":
+        assert f" netloc={netloc}" in r

parsl/tests/unit/test_usage_tracking.py

@@ -0,0 +1,66 @@
+"""Test usage_tracking values."""
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.errors import ConfigurationError
+
+
+@pytest.mark.local
+def test_config_load():
+    """Test loading a config with usage tracking."""
+    with parsl.load(Config(usage_tracking=3)):
+        pass
+    parsl.clear()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", (0, 1, 2, 3, False, True))
+def test_valid(level):
+    """Test valid usage_tracking values."""
+    Config(usage_tracking=level)
+    assert Config(usage_tracking=level).usage_tracking == level
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", (12, 1000, -1))
+def test_invalid_values(level):
+    """Test invalid usage_tracking values."""
+    with pytest.raises(ConfigurationError):
+        Config(usage_tracking=level)
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", ("abcd", None, bytes(1), 1.0, 1j, object()))
+def test_invalid_types(level):
+    """Test invalid usage_tracking types."""
+    with pytest.raises(Exception) as ex:
+        Config(usage_tracking=level)
+
+    # with typeguard 4.x this is TypeCheckError,
+    # with typeguard 2.x this is TypeError
+    # we can't instantiate TypeCheckError if we're in typeguard 2.x environment
+    # because it does not exist... so check name using strings.
+    assert ex.type.__name__ in ["TypeCheckError", "TypeError"]
+
+
+@pytest.mark.local
+def test_valid_project_name():
+    """Test valid project_name."""
+    assert (
+        Config(
+            usage_tracking=3,
+            project_name="unit-test",
+        ).project_name == "unit-test"
+    )
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("name", (1, 1.0, True, object()))
+def test_invalid_project_name(name):
+    """Test invalid project_name."""
+    with pytest.raises(Exception) as ex:
+        Config(usage_tracking=3, project_name=name)
+
+    assert ex.type.__name__ in ["TypeCheckError", "TypeError"]

parsl/usage_tracking/api.py

@@ -0,0 +1,65 @@
+import inspect
+from abc import abstractmethod
+from functools import singledispatch
+from typing import Any, List, Sequence
+
+from parsl.utils import RepresentationMixin
+
+# Traverse the configuration hierarchy, returning a JSON component
+# for each one. Configuration components which implement
+# RepresentationMixin will be in the right form for inspecting
+# object attributes. Configuration components which are lists or tuples
+# are traversed in sequence. Other types default to reporting no
+# usage information.
+
+
+@singledispatch
+def get_parsl_usage(obj) -> List[Any]:
+    return []
+
+
+@get_parsl_usage.register
+def get_parsl_usage_representation_mixin(obj: RepresentationMixin) -> List[Any]:
+    t = type(obj)
+    qualified_name = t.__module__ + "." + t.__name__
+
+    # me can contain anything that can be rendered as JSON
+    me: List[Any] = []
+
+    if isinstance(obj, UsageInformation):
+        # report rich usage information for this component
+        attrs = {'c': qualified_name}
+        attrs.update(obj.get_usage_information())
+        me = [attrs]
+    else:
+        # report the class name of this component
+        me = [qualified_name]
+
+    # unwrap typeguard-style unwrapping
+    init: Any = type(obj).__init__
+    if hasattr(init, '__wrapped__'):
+        init = init.__wrapped__
+
+    argspec = inspect.getfullargspec(init)
+
+    for arg in argspec.args[1:]:  # skip first arg, self
+        arg_value = getattr(obj, arg)
+        d = get_parsl_usage(arg_value)
+        me += d
+
+    return me
+
+
+@get_parsl_usage.register(list)
+@get_parsl_usage.register(tuple)
+def get_parsl_usage_sequence(obj: Sequence) -> List[Any]:
+    result = []
+    for v in obj:
+        result += get_parsl_usage(v)
+    return result
+
+
+class UsageInformation:
+    @abstractmethod
+    def get_usage_information(self) -> dict:
+        pass

parsl/usage_tracking/levels.py

@@ -0,0 +1,6 @@
+"""Module for defining the usage tracking levels."""
+
+DISABLED = 0  # Tracking is disabled
+LEVEL_1 = 1  # Share info about Parsl version, Python version, platform
+LEVEL_2 = 2  # Share info about config + level 1
+LEVEL_3 = 3  # Share info about app count, app fails, execution time + level 2