parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_staging/test_docs_1.py

@@ -1,6 +1,6 @@
 import pytest
 
-from parsl import python_app, File
+from parsl import File, python_app
 
 
 @python_app
@@ -12,6 +12,7 @@ def convert(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test():
     # create an remote Parsl file
     inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')

parsl/tests/test_staging/test_docs_2.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl import bash_app, File
+
+from parsl import File, bash_app
 from parsl.tests.configs.local_threads import fresh_config as local_config
 
 

parsl/tests/test_staging/test_elaborate_noop_file.py

@@ -6,15 +6,15 @@
 # of the globus staging provider
 
 import logging
+
 import pytest
 
 import parsl
-
 from parsl import bash_app, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.tests.test_staging.staging_provider import NoOpTestingFileStaging, NoOpError
+from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,6 @@ def storage_access_parsl():
     yield _setup_config
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local

parsl/tests/{test_data → test_staging}/test_file.py

@@ -22,11 +22,11 @@ def test_files():
 
 
 @pytest.mark.local
-def test_open():
-    with open('test-open.txt', 'w') as tfile:
-        tfile.write('Hello')
+def test_open(tmpd_cwd):
+    fpath = tmpd_cwd / 'test-open.txt'
+    fpath.write_text('Hello')
 
-    pfile = File('test-open.txt')
+    pfile = File(fpath)
 
-    with open(str(pfile), 'r') as opfile:
-        assert (opfile.readlines()[0] == 'Hello')
+    with open(pfile) as opfile:
+        assert (opfile.read() == 'Hello')

parsl/tests/{test_data → test_staging}/test_output_chain_filenames.py

@@ -1,5 +1,7 @@
 from concurrent.futures import Future
 
+import pytest
+
 from parsl import File
 from parsl.app.app import bash_app
 
@@ -14,6 +16,7 @@ def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
     return f"echo '{inputs[0]}' > {outputs[0]}"
 
 
+@pytest.mark.shared_fs
 def test_behavior(tmpd_cwd):
     expected_path = str(tmpd_cwd / "simple-out.txt")
     app1_future = app1(

parsl/tests/test_staging/test_staging_ftp.py

@@ -15,6 +15,7 @@ def sort_strings(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_ftp():
     """Test staging for an ftp file
 

parsl/tests/test_staging/test_staging_https.py

@@ -1,9 +1,9 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 
-import pytest
-
 # This config is for the local test which will adding an executor.
 # Most tests in this file should be non-local and use the configuration
 # specificed with --config, not this one.
@@ -48,6 +48,7 @@ def sort_strings_additional_executor(inputs=(), outputs=()):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_cleannet(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -68,6 +69,7 @@ def test_staging_https_local(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_kwargs(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -78,6 +80,7 @@ def test_staging_https_kwargs(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_args(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')

parsl/tests/test_staging/test_staging_stdout.py

@@ -0,0 +1,64 @@
+import logging
+import os
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.bash_app
+def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello ; echo goodbye >&2"
+
+
+@pytest.mark.staging_required
+def test_stdout_staging_file(tmpd_cwd, caplog):
+    basename = str(tmpd_cwd) + "/stdout.txt"
+    stdout_file = File("file://" + basename)
+
+    app_future = output_to_stds(stdout=stdout_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    assert os.path.exists(basename)
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+
+@pytest.mark.staging_required
+def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
+    zipfile_name = str(tmpd_cwd) + "/staging.zip"
+    stdout_relative_path = "somewhere/test-out.txt"
+    stdout_file = File("zip:" + zipfile_name + "/" + stdout_relative_path)
+
+    stderr_relative_path = "somewhere/test-error.txt"
+    stderr_file = File("zip:" + zipfile_name + "/" + stderr_relative_path)
+
+    app_future = output_to_stds(stdout=stdout_file, stderr=stderr_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    # check the file exists as soon as possible
+    assert os.path.exists(zipfile_name)
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stdout_relative_path) as f:
+            assert f.readlines() == [b'hello\n']
+
+    assert isinstance(app_future.stderr, DataFuture)
+    app_future.stderr.result()
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stderr_relative_path) as f:
+            # The last line of stderr should be goodbye, but Parsl will write
+            # other Parsl-specific into to stderr before that, so only assert
+            # the behaviour of the final line.
+            assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_staging/test_zip_in.py

@@ -0,0 +1,39 @@
+import random
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_in(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    # create a zip file containing one file with some abitrary number of lines
+    n_lines = random.randint(0, 1000)
+
+    with zipfile.ZipFile(zip_path, mode='w') as z:
+        with z.open(file_base, mode='w') as f:
+            for _ in range(n_lines):
+                f.write(b'someline\n')
+
+    app_future = count_lines(zip_file)
+
+    assert app_future.result() == n_lines

parsl/tests/test_staging/test_zip_out.py

@@ -0,0 +1,110 @@
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@pytest.mark.local
+def test_zip_path_split():
+    from parsl.data_provider.zip import zip_path_split
+    assert zip_path_split("/tmp/foo/this.zip/inside/here.txt") == ("/tmp/foo/this.zip", "inside/here.txt")
+
+
+@parsl.bash_app
+def output_something(outputs=()):
+    """This should output something into every specified output file:
+    the position in the output sequence will be written into the
+    corresponding output file.
+    """
+    cmds = []
+    for n in range(len(outputs)):
+        cmds.append(f"echo {n} > {outputs[n]}")
+
+    return "; ".join(cmds)
+
+
+@pytest.mark.local
+def test_zip_out(tmpd_cwd):
+    # basic test of zip file stage-out
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip:{zip_path / file_base}")
+
+    app_future = output_something(outputs=[of])
+    output_file_future = app_future.outputs[0]
+
+    app_future.result()
+    output_file_future.result()
+
+    assert zipfile.is_zipfile(zip_path)
+
+    with zipfile.ZipFile(zip_path) as z:
+        assert file_base in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(file_base) as f:
+            assert f.readlines() == [b'0\n']
+
+
+@pytest.mark.local
+def test_zip_out_multi(tmpd_cwd):
+    # tests multiple files, multiple zip files and multiple
+    # sub-paths
+
+    zip_path_1 = tmpd_cwd / "container1.zip"
+    zip_path_2 = tmpd_cwd / "container2.zip"
+
+    relative_file_path_1 = "a/b/c/data.txt"
+    relative_file_path_2 = "something.txt"
+    relative_file_path_3 = "a/d/other.txt"
+    of1 = File(f"zip:{zip_path_1 / relative_file_path_1}")
+    of2 = File(f"zip:{zip_path_1 / relative_file_path_2}")
+    of3 = File(f"zip:{zip_path_2 / relative_file_path_3}")
+
+    app_future = output_something(outputs=[of1, of2, of3])
+
+    for f in app_future.outputs:
+        f.result()
+
+    app_future.result()
+
+    assert zipfile.is_zipfile(zip_path_1)
+
+    with zipfile.ZipFile(zip_path_1) as z:
+        assert relative_file_path_1 in z.namelist()
+        assert relative_file_path_2 in z.namelist()
+        assert len(z.namelist()) == 2
+        with z.open(relative_file_path_1) as f:
+            assert f.readlines() == [b'0\n']
+        with z.open(relative_file_path_2) as f:
+            assert f.readlines() == [b'1\n']
+
+    assert zipfile.is_zipfile(zip_path_2)
+
+    with zipfile.ZipFile(zip_path_2) as z:
+        assert relative_file_path_3 in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(relative_file_path_3) as f:
+            assert f.readlines() == [b'2\n']
+
+
+@pytest.mark.local
+def test_zip_bad_authority(tmpd_cwd):
+    # tests that there's an exception when staging a ZIP url with an authority
+    # section specified, rather than silently ignoring it. This simulates a
+    # user who misunderstands what that piece of what a zip: URL means.
+
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip://someauthority/{zip_path / file_base}")
+
+    with pytest.raises(ZipAuthorityError):
+        output_something(outputs=[of])

parsl/tests/test_staging/test_zip_to_zip.py

@@ -0,0 +1,41 @@
+import random
+import zipfile
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def generate_lines(n: int, *, outputs):
+    with open(outputs[0], "w") as f:
+        for x in range(n):
+            # write numbered lines
+            f.write(str(x) + "\n")
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_pipeline(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    n_lines = random.randint(0, 1000)
+    generate_fut = generate_lines(n_lines, outputs=[zip_file])
+    n_lines_out = count_lines(generate_fut.outputs[0]).result()
+
+    assert n_lines == n_lines_out

parsl/tests/test_summary.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl.tests.configs.local_threads import fresh_config
 
 
@@ -22,7 +23,6 @@ def test_summary(caplog):
     fail().exception()
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     assert "Summary of tasks in DFK:" in caplog.text
     assert "Tasks in state States.exec_done: 1" in caplog.text

parsl/tests/test_thread_parallelism.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_threads/test_configs.py

@@ -7,9 +7,9 @@ from parsl.tests.configs.local_threads import fresh_config
 
 @python_app
 def worker_identify(x, sleep_dur=0.2):
-    import time
     import os
     import threading
+    import time
     time.sleep(sleep_dur)
     return {"pid": os.getpid(),
             "tid": threading.current_thread()}
@@ -30,5 +30,4 @@ def test_parallel_for():
     assert thread_count <= config.executors[0].max_threads, "More threads than allowed"
     assert process_count == 1, "More processes than allowed"
     dfk.cleanup()
-    parsl.clear()
     return d

parsl/tests/test_threads/test_lazy_errors.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl import python_app
 from parsl.tests.configs.local_threads import fresh_config
 
@@ -24,5 +25,4 @@ def test_lazy_behavior():
         assert f.done()
 
     parsl.dfk().cleanup()
-    parsl.clear()
     return

parsl/tests/test_utils/test_execute_wait.py

@@ -0,0 +1,35 @@
+import pytest
+
+from parsl.utils import execute_wait
+
+
+@pytest.mark.local
+def test_env():
+    ''' Regression testing for issue #27
+    '''
+
+    rc, stdout, stderr = execute_wait("env", 1)
+
+    stdout = stdout.split('\n')
+    x = [s for s in stdout if s.startswith("PATH=")]
+    assert x, "PATH not found"
+
+    x = [s for s in stdout if s.startswith("HOME=")]
+    assert x, "HOME not found"
+
+
+@pytest.mark.local
+def test_large_output_2210():
+    """Regression test for #2210.
+    execute_wait was hanging if the specified command gave too
+    much output, due to a race condition between process exiting and
+    pipes filling up.
+    """
+
+    # this will output 128kb of stdout
+    execute_wait("yes | dd count=128 bs=1024", walltime=60)
+
+    # if this test fails, execute_wait should raise a timeout
+    # exception.
+
+    # The contents out the output is not verified by this test

parsl/tests/test_utils/test_sanitize_dns.py

@@ -0,0 +1,76 @@
+import random
+import re
+
+import pytest
+
+from parsl.utils import sanitize_dns_label_rfc1123, sanitize_dns_subdomain_rfc1123
+
+# Ref: https://datatracker.ietf.org/doc/html/rfc1123
+DNS_LABEL_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$'
+DNS_SUBDOMAIN_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$'
+
+test_labels = [
+    "example-label-123",           # Valid label
+    "EXAMPLE",                     # Case sensitivity
+    "!@#example*",                 # Remove invalid characters
+    "--leading-and-trailing--",    # Leading and trailing hyphens
+    "..leading.and.trailing..",    # Leading and tailing dots
+    "multiple..dots",              # Consecutive dots
+    "valid--label",                # Consecutive hyphens
+    "a" * random.randint(64, 70),  # Longer than 63 characters
+    f"{'a' * 62}-a",               # Trailing hyphen at max length
+]
+
+
+def _generate_test_subdomains(num_subdomains: int):
+    subdomains = []
+    for _ in range(num_subdomains):
+        num_labels = random.randint(1, 5)
+        labels = [test_labels[random.randint(0, num_labels - 1)] for _ in range(num_labels)]
+        subdomain = ".".join(labels)
+        subdomains.append(subdomain)
+    return subdomains
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", test_labels)
+def test_sanitize_dns_label_rfc1123(raw_string: str):
+    print(sanitize_dns_label_rfc1123(raw_string))
+    assert re.match(DNS_LABEL_REGEX, sanitize_dns_label_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", "-", "@", "$$$"))
+def test_sanitize_dns_label_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_label_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS label is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", _generate_test_subdomains(10))
+def test_sanitize_dns_subdomain_rfc1123(raw_string: str):
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("char", ("-", "."))
+def test_sanitize_dns_subdomain_rfc1123_trailing_non_alphanumeric_at_max_length(char: str):
+    raw_string = (f"{'a' * 61}." * 4) + f".aaaa{char}a"
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", ".", "..."))
+def test_sanitize_dns_subdomain_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_subdomain_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS subdomain is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "raw_string", ("a" * 253, "a" * random.randint(254, 300)), ids=("254 chars", ">253 chars")
+)
+def test_sanitize_dns_subdomain_rfc1123_max_length(raw_string: str):
+    assert len(sanitize_dns_subdomain_rfc1123(raw_string)) <= 253

parsl/tests/unit/test_address.py

@@ -0,0 +1,20 @@
+import pytest
+
+from parsl.addresses import tcp_url
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("address, port,expected", [
+    ("127.0.0.1", 55001, "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", "55001", "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", None, "tcp://127.0.0.1"),
+    ("::1", "55001", "tcp://[::1]:55001"),
+    ("::ffff:127.0.0.1", 55001, "tcp://[::ffff:127.0.0.1]:55001"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("*", None, "tcp://*"),
+])
+def test_tcp_url(address, port, expected):
+    """Confirm valid address generation"""
+    result = tcp_url(address, port)
+    assert result == expected

parsl/tests/unit/test_file.py

@@ -0,0 +1,99 @@
+import os
+from unittest import mock
+
+import pytest
+
+from parsl import File
+
+_MOCK_BASE = "parsl.data_provider.files."
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("http", "https", "ftp", "ftps", "asdfasdf"))
+def test_file_init_scheme(scheme):
+    basename = "some_base_name"
+    path = f"/some/path/1/2/3/{basename}"
+    fqdn = "some.fqdn.example.com"
+    exp_url = f"{scheme}://{fqdn}{path}"
+    f = File(exp_url)
+    assert f.url == exp_url, "Expected given url to be stored"
+    assert f.scheme == scheme
+    assert f.netloc == fqdn
+    assert f.path == path
+    assert f.filename == basename
+    assert f.local_path is None, "Expect only set by API consumer, not constructor"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("url", ("some weird :// url", "", "a"))
+def test_file_init_file_url_fallback(url):
+    exp_url = "some weird :// url"
+    f = File(exp_url)
+    assert f.url == exp_url
+    assert not f.netloc, "invalid host, should be no netloc"
+    assert f.path == exp_url, "Should fail to fully parse, so path is whole url"
+    assert f.filename == exp_url.rsplit("/", 1)[-1]
+
+    assert f.scheme == "file"
+
+
+@pytest.mark.local
+def test_file_proxies_for_filepath(randomstring):
+    # verify (current) expected internal hookup
+    exp_filepath = randomstring()
+    with mock.patch(
+        f"{_MOCK_BASE}File.filepath", new_callable=mock.PropertyMock
+    ) as mock_fpath:
+        mock_fpath.return_value = exp_filepath
+        f = File("")
+        assert str(f) == exp_filepath
+        assert os.fspath(f) == exp_filepath
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("file://", ""))
+def test_file_filepath_local_path_is_priority(scheme, randomstring):
+    exp_path = "/some/local/path"
+    url = f"{scheme}{exp_path}"
+    f = File(url)
+
+    f.local_path = randomstring()
+    assert f.filepath == f.local_path
+
+    f.local_path = None
+    assert f.filepath == exp_path
+
+
+@pytest.mark.local
+def test_file_filepath_requires_local_accessible_path():
+    with pytest.raises(ValueError) as pyt_exc:
+        _ = File("http://").filepath
+
+    assert "No local_path" in str(pyt_exc.value), "Expected reason in exception"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("scheme", ("https", "ftps", "", "file", "asdfasdf"))
+def test_file_repr(scheme):
+    netloc = "some.netloc"
+    filename = "some_file_name"
+    path = f"/some/path/{filename}"
+    if scheme:
+        url = f"{scheme}://{netloc}{path}"
+    else:
+        scheme = "file"
+        url = path
+
+    f = File(url)
+    r = repr(f)
+    assert r.startswith("<")
+    assert r.endswith(">")
+    assert f"<{type(f).__name__} " in r
+    assert f" at 0x{id(f):x}" in r
+    assert f" url={url}" in r
+    assert f" scheme={scheme}" in r
+    assert f" path={path}" in r
+    assert f" filename={filename}" in r
+
+    if scheme != "file":
+        assert f" netloc={netloc}" in r