parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/configs/frontera.py

@@ -1,10 +1,10 @@
 from parsl.config import Config
-from parsl.channels import LocalChannel
-from parsl.providers import SlurmProvider
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 from .user_opts import user_opts
+
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
 """
@@ -19,7 +19,6 @@ def fresh_config():
                 encrypted=True,
                 provider=SlurmProvider(
                     cmd_timeout=60,     # Add extra time for slow scheduler responses
-                    channel=LocalChannel(),
                     nodes_per_block=2,
                     init_blocks=1,
                     min_blocks=1,

parsl/tests/configs/htex_local.py

@@ -1,9 +1,7 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def fresh_config():
@@ -11,11 +9,11 @@ def fresh_config():
         executors=[
             HighThroughputExecutor(
                 label="htex_local",
+                loopback_address="::1",
                 worker_debug=True,
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                     launcher=SimpleLauncher(),

parsl/tests/configs/htex_local_alternate.py

@@ -15,22 +15,19 @@ will cause substantially different behaviour on whatever
 those timing parameters control.
 """
 
-# imports for monitoring:
-from parsl.monitoring import MonitoringHub
-
 import os
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
-
 from parsl.config import Config
+from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.data_provider.zip import ZipFileStaging
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
 
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
-from parsl.data_provider.file_noop import NoOpFileStaging
+# imports for monitoring:
+from parsl.monitoring import MonitoringHub
+from parsl.providers import LocalProvider
 
 working_dir = os.getcwd() + "/" + "test_htex_alternate"
 
@@ -42,7 +39,7 @@ def fresh_config():
                 address="127.0.0.1",
                 label="htex_Local",
                 working_dir=working_dir,
-                storage_access=[FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()],
+                storage_access=[ZipFileStaging(), FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()],
                 worker_debug=True,
                 cores_per_worker=1,
                 heartbeat_period=2,
@@ -50,7 +47,6 @@ def fresh_config():
                 poll_period=100,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     min_blocks=0,
                     max_blocks=5,
@@ -64,11 +60,11 @@ def fresh_config():
         retries=2,
         monitoring=MonitoringHub(
                         hub_address="localhost",
-                        hub_port=55055,
                         monitoring_debug=False,
                         resource_monitoring_interval=1,
         ),
-        usage_tracking=True
+        usage_tracking=3,
+        project_name="parsl htex_local_alternate test configuration"
     )
 
 

parsl/tests/configs/htex_local_intask_staging.py

@@ -1,13 +1,10 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
-from parsl.data_provider.file_noop import NoOpFileStaging
-
 from parsl.config import Config
+from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[
@@ -17,7 +14,6 @@ config = Config(
             cores_per_worker=1,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 launcher=SimpleLauncher(),

parsl/tests/configs/htex_local_rsync_staging.py

@@ -1,13 +1,10 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
-from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-
-from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[
@@ -18,7 +15,6 @@ config = Config(
             working_dir="./rsync-workdir/",
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 launcher=SimpleLauncher(),

parsl/tests/configs/local_radical.py

@@ -1,9 +1,7 @@
 import os
 
 from parsl.config import Config
-from parsl.executors.radical import RadicalPilotExecutor
-from parsl.executors.radical import ResourceConfig
-
+from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
 
 rpex_cfg = ResourceConfig()
 

parsl/tests/configs/local_radical_mpi.py

@@ -1,10 +1,10 @@
 import os
+
 from parsl.config import Config
 
 
 def fresh_config():
-    from parsl.executors.radical import ResourceConfig
-    from parsl.executors.radical import RadicalPilotExecutor
+    from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
 
     rpex_cfg = ResourceConfig()
     rpex_cfg.worker_type = "MPI"

parsl/tests/configs/local_threads_checkpoint_periodic.py

@@ -1,13 +1,11 @@
 from parsl.config import Config
 from parsl.executors.threads import ThreadPoolExecutor
 
-config = Config(
-    executors=[
-        ThreadPoolExecutor(
-            label='local_threads_checkpoint_periodic',
-            max_threads=1
-        )
-    ],
-    checkpoint_mode='periodic',
-    checkpoint_period='00:00:05'
-)
+
+def fresh_config():
+    tpe = ThreadPoolExecutor(label='local_threads_checkpoint_periodic', max_threads=1)
+    return Config(
+        executors=[tpe],
+        checkpoint_mode='periodic',
+        checkpoint_period='00:00:02'
+    )

parsl/tests/configs/local_threads_monitoring.py

@@ -5,7 +5,6 @@ from parsl.monitoring import MonitoringHub
 config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
                 monitoring=MonitoringHub(
                     hub_address="localhost",
-                    hub_port=55055,
                     resource_monitoring_interval=3,
                 )
                 )

parsl/tests/configs/midway.py

@@ -1,7 +1,7 @@
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 from .user_opts import user_opts
 

parsl/tests/configs/nscc_singapore.py

@@ -1,8 +1,8 @@
-from parsl.providers import PBSProProvider
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import MpiRunLauncher
 from parsl.addresses import address_by_interface
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import MpiRunLauncher
+from parsl.providers import PBSProProvider
 
 from .user_opts import user_opts
 

parsl/tests/configs/osg_htex.py

@@ -1,6 +1,6 @@
 from parsl.config import Config
-from parsl.providers import CondorProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import CondorProvider
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
 # If you are a user copying-and-pasting this as an example, make sure to either

parsl/tests/configs/petrelkube.py

@@ -1,10 +1,11 @@
+import os
+
+from parsl.addresses import address_by_route
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
-from parsl.addresses import address_by_route
 
 from .user_opts import user_opts
-import os
 
 
 def fresh_config():

parsl/tests/configs/slurm_local.py

@@ -0,0 +1,24 @@
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+
+
+def fresh_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="docker_slurm",
+                encrypted=True,
+                provider=SlurmProvider(
+                    cmd_timeout=60,     # Add extra time for slow scheduler responses
+                    nodes_per_block=1,
+                    init_blocks=1,
+                    min_blocks=1,
+                    max_blocks=1,
+                    walltime='00:10:00',
+                    launcher=SrunLauncher(),
+                ),
+            )
+        ],
+    )

parsl/tests/configs/summit.py

@@ -4,6 +4,7 @@ from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
 
 from .user_opts import user_opts
+
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
 """

parsl/tests/configs/taskvine_ex.py

@@ -1,13 +1,10 @@
 from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
 
 
 def fresh_config():
     return Config(executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(port=9000),
-                                              worker_launch_method='factory',
-                                              storage_access=[FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()])])
+                                              worker_launch_method='factory')])

parsl/tests/configs/user_opts.py

@@ -60,7 +60,8 @@ user_opts = {
     #     'username': OSG_USERNAME,
     #     'script_dir': '/home/{}/parsl_scripts'.format(OSG_USERNAME),
     #     'scheduler_options': "",
-    #     'worker_init' : 'module load python/3.5.2; python3 -m venv parsl_env; source parsl_env/bin/activate; python3 -m pip install parsl==0.5.2'
+    #     'worker_init' : 'module load python/3.5.2; python3 -m venv parsl_env;
+    #                      source parsl_env/bin/activate; python3 -m pip install parsl==0.5.2'
     # },
     # 'swan': {
     #     'username': SWAN_USERNAME,
@@ -134,13 +135,6 @@ user_opts = {
     #     # For example:
     #     'remote_writeable': 'globus://af7bda53-6d04-11e5-ba46-22000b92c6ec/home/bzc/'
     # },
-    # 'adhoc': {
-    #    # This specifies configuration parameters when testing an ad-hoc SSH based cluster
-    #    'username': 'fixme', # username on remote systems
-    #    'remote_hostnames': ['hostname1', 'hostname2'], # addresses of remote systems
-    #    'worker_init': 'init commands',  # worker_init for remote systems
-    #    'script_dir': "/path"  # script directory on remote systems
-    # }
     #
 }  # type: Dict[str, Any]
 

parsl/tests/configs/workqueue_ex.py

@@ -1,12 +1,10 @@
 from parsl.config import Config
-from parsl.executors import WorkQueueExecutor
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.executors import WorkQueueExecutor
 
 
 def fresh_config():
     return Config(executors=[WorkQueueExecutor(port=9000,
-                                               coprocess=True,
-                                               storage_access=[FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()])])
+                                               coprocess=True)])

parsl/tests/conftest.py

@@ -3,22 +3,24 @@ import itertools
 import logging
 import os
 import pathlib
+import random
 import re
 import shutil
-import time
-import types
 import signal
+import string
 import sys
 import tempfile
 import threading
+import time
 import traceback
+import types
 import typing as t
 from datetime import datetime
 from glob import glob
 from itertools import chain
 
-import pytest
 import _pytest.runner as runner
+import pytest
 
 import parsl
 from parsl.dataflow.dflow import DataFlowKernelLoader
@@ -56,7 +58,7 @@ def tmpd_cwd_session(pytestconfig):
 
     config = re.sub(r"[^A-z0-9_-]+", "_", pytestconfig.getoption('config')[0])
     cwd = pathlib.Path(os.getcwd())
-    pytest_dir = cwd / ".pytest"
+    pytest_dir = cwd / "pytest-parsl"
     pytest_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
 
     test_dir_prefix = "parsltest-"
@@ -135,28 +137,31 @@ def pytest_configure(config):
     )
     config.addinivalue_line(
         'markers',
-        'noci: mark test to be unsuitable for running during automated tests'
+        'cleannet: Enable tests that require a clean network connection (such as for testing FTP)'
     )
-
     config.addinivalue_line(
         'markers',
-        'cleannet: Enable tests that require a clean network connection (such as for testing FTP)'
+        'staging_required: Marks tests that require a staging provider, when there is no sharedFS'
     )
     config.addinivalue_line(
         'markers',
-        'issue363: Marks tests that require a shared filesystem for stdout/stderr - see issue #363'
+        'multiple_cores_required: Marks tests that require multiple cores, such as htex affinity'
     )
     config.addinivalue_line(
         'markers',
-        'staging_required: Marks tests that require a staging provider, when there is no sharedFS)'
+        'unix_filesystem_permissions_required: Marks tests that require unix-level filesystem permission enforcement'
     )
     config.addinivalue_line(
         'markers',
-        'sshd_required: Marks tests that require a SSHD'
+        'issue3328: Marks tests broken by issue #3328'
     )
     config.addinivalue_line(
         'markers',
-        'multiple_cores_required: Marks tests that require multiple cores, such as htex affinity'
+        'executor_supports_std_stream_tuples: Marks tests that require tuple support for stdout/stderr'
+    )
+    config.addinivalue_line(
+        'markers',
+        'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
     )
 
 
@@ -200,7 +205,7 @@ def load_dfk_session(request, pytestconfig, tmpd_cwd_session):
         if parsl.dfk() != dfk:
             raise RuntimeError("DFK changed unexpectedly during test")
         dfk.cleanup()
-        parsl.clear()
+        assert DataFlowKernelLoader._dfk is None
     else:
         yield
 
@@ -246,12 +251,13 @@ def load_dfk_local_module(request, pytestconfig, tmpd_cwd_session):
 
         if callable(local_teardown):
             local_teardown()
+            assert DataFlowKernelLoader._dfk is None, "Expected teardown to clear DFK"
 
         if local_config:
             if parsl.dfk() != dfk:
                 raise RuntimeError("DFK changed unexpectedly during test")
             dfk.cleanup()
-            parsl.clear()
+            assert DataFlowKernelLoader._dfk is None
 
     else:
         yield
@@ -422,3 +428,11 @@ def try_assert():
             raise AssertionError("Bad assert call: no attempts or timeout period")
 
     yield _impl
+
+
+@pytest.fixture
+def randomstring():
+    def func(length=5, alphabet=string.ascii_letters):
+        return "".join(random.choice(alphabet) for _ in range(length))
+
+    return func

parsl/tests/integration/test_stress/test_python_simple.py

@@ -1,10 +1,9 @@
+import argparse
+import time
+
 import parsl
 from parsl import python_app
 from parsl.configs.htex_local import config
-# from parsl.configs.local_threads import config
-
-import time
-import argparse
 
 
 @python_app

parsl/tests/integration/test_stress/test_python_threads.py

@@ -1,14 +1,12 @@
 ''' Testing bash apps
 '''
-import parsl
-from parsl import python_app
-
-import time
 import argparse
+import time
 
+import parsl
+from parsl import python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/manual_tests/htex_local.py

@@ -1,11 +1,10 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-# from parsl.launchers import SimpleLauncher
-from parsl.launchers import SingleNodeLauncher
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 
+# from parsl.launchers import SimpleLauncher
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
+
 config = Config(
     executors=[
         HighThroughputExecutor(
@@ -15,7 +14,6 @@ config = Config(
             cores_per_worker=1,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 # tasks_per_node=1,  # For HighThroughputExecutor, this option should in most cases be 1

parsl/tests/manual_tests/test_basic.py

@@ -2,6 +2,7 @@ import argparse
 import time
 
 import parsl
+
 # Tested. Confirmed. Local X Local X SingleNodeLauncher
 # from parsl.tests.configs.local_ipp import config
 

parsl/tests/manual_tests/test_log_filter.py

@@ -1,6 +1,8 @@
 import argparse
-import parsl
 import logging
+
+import parsl
+
 parsl.load()
 
 from parsl import python_app

parsl/tests/manual_tests/test_memory_limits.py

@@ -1,15 +1,14 @@
 import argparse
-import parsl
-import psutil
 import multiprocessing
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
+import psutil
 
+import parsl
 from parsl.app.app import python_app  # , bash_app
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 
 @python_app
@@ -30,7 +29,6 @@ def test_simple(mem_per_worker):
                 suppress_failure=True,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                     launcher=SingleNodeLauncher(),

parsl/tests/manual_tests/test_regression_220.py

@@ -1,7 +1,8 @@
+import time
+
 import pytest
 
 from parsl import DataFlowKernel, set_stream_logger
-import time
 from parsl.tests.configs.local_threads import config
 
 

parsl/tests/manual_tests/test_udp_simple.py

@@ -1,9 +1,10 @@
+import logging
+
+import parsl
 from parsl import python_app
-from parsl.monitoring.monitoring import MonitoringHub
 from parsl.config import Config
 from parsl.executors import ThreadPoolExecutor
-import parsl
-import logging
+from parsl.monitoring.monitoring import MonitoringHub
 
 
 def local_setup():
@@ -14,7 +15,6 @@ def local_setup():
         ],
         monitoring=MonitoringHub(
             hub_address="127.0.0.1",
-            hub_port=55055,
             logging_level=logging.INFO,
             resource_monitoring_interval=10))
 

parsl/tests/manual_tests/test_worker_count.py

@@ -1,7 +1,7 @@
 import argparse
-import time
 import math
 import multiprocessing
+import time
 
 import parsl
 
@@ -10,10 +10,11 @@ CORES_PER_WORKER = 1
 EXPECTED_WORKERS = math.floor(CORES / CORES_PER_WORKER)
 
 
+from parsl.executors import HighThroughputExecutor
+
 # from parsl.tests.configs.htex_local import config
 from parsl.tests.manual_tests.htex_local import config
 
-from parsl.executors import HighThroughputExecutor
 assert isinstance(config.executors[0], HighThroughputExecutor)
 config.executors[0].cores_per_worker = CORES_PER_WORKER
 config.executors[0].provider.init_blocks = 1

parsl/tests/scaling_tests/htex_local.py

@@ -1,8 +1,7 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import LocalProvider
+
 # import os
 config = Config(
     executors=[
@@ -12,7 +11,6 @@ config = Config(
             max_workers_per_node=8,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
             ),