parsl 2024.12.2__py3-none-any.whl → 2024.12.16__py3-none-any.whl

parsl/providers/slurm/slurm.py

@@ -3,12 +3,10 @@ import math
 import os
 import re
 import time
-from typing import Optional
+from typing import Any, Dict, Optional
 
 import typeguard
 
-from parsl.channels import LocalChannel
-from parsl.channels.base import Channel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
@@ -73,8 +71,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
     clusters : str
         Slurm cluster name, or comma seperated cluster list, used to choose between different clusters in a federated Slurm instance.
         If unspecified or ``None``, no slurm directive for clusters will be added.
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_node : int
@@ -119,7 +115,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                  qos: Optional[str] = None,
                  constraint: Optional[str] = None,
                  clusters: Optional[str] = None,
-                 channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_node: Optional[int] = None,
                  mem_per_node: Optional[int] = None,
@@ -136,7 +131,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                  launcher: Launcher = SingleNodeLauncher()):
         label = 'slurm'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -286,8 +280,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
 
         logger.debug("Requesting one block with {} nodes".format(self.nodes_per_block))
 
-        job_config = {}
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config: Dict[str, Any] = {}
+        job_config["submit_script_dir"] = self.script_dir
         job_config["nodes"] = self.nodes_per_block
         job_config["tasks_per_node"] = tasks_per_node
         job_config["walltime"] = wtime_to_minutes(self.walltime)

parsl/providers/torque/torque.py

@@ -2,7 +2,6 @@ import logging
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import AprunLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -33,8 +32,6 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str
@@ -65,7 +62,6 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
 
     """
     def __init__(self,
-                 channel=LocalChannel(),
                  account=None,
                  queue=None,
                  scheduler_options='',
@@ -80,7 +76,6 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
                  cmd_timeout=120):
         label = 'torque'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -170,8 +165,7 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
                      tasks_per_node)
 
         job_config = {}
-        # TODO : script_path might need to change to accommodate script dir set via channels
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config["submit_script_dir"] = self.script_dir
         job_config["nodes"] = self.nodes_per_block
         job_config["task_blocks"] = self.nodes_per_block * tasks_per_node
         job_config["nodes_per_block"] = self.nodes_per_block

parsl/tests/configs/cc_in2p3.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
@@ -14,7 +13,6 @@ def fresh_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=GridEngineProvider(
-                    channel=LocalChannel(),
                     nodes_per_block=2,
                     init_blocks=2,
                     max_blocks=2,

parsl/tests/configs/frontera.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
@@ -20,7 +19,6 @@ def fresh_config():
                 encrypted=True,
                 provider=SlurmProvider(
                     cmd_timeout=60,     # Add extra time for slow scheduler responses
-                    channel=LocalChannel(),
                     nodes_per_block=2,
                     init_blocks=1,
                     min_blocks=1,

parsl/tests/configs/htex_local.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -15,7 +14,6 @@ def fresh_config():
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                     launcher=SimpleLauncher(),

parsl/tests/configs/htex_local_alternate.py

@@ -17,7 +17,6 @@ those timing parameters control.
 
 import os
 
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.file_noop import NoOpFileStaging
 from parsl.data_provider.ftp import FTPInTaskStaging
@@ -48,7 +47,6 @@ def fresh_config():
                 poll_period=100,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     min_blocks=0,
                     max_blocks=5,

parsl/tests/configs/htex_local_intask_staging.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.file_noop import NoOpFileStaging
 from parsl.data_provider.ftp import FTPInTaskStaging
@@ -15,7 +14,6 @@ config = Config(
             cores_per_worker=1,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 launcher=SimpleLauncher(),

parsl/tests/configs/htex_local_rsync_staging.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
 from parsl.data_provider.http import HTTPInTaskStaging
@@ -16,7 +15,6 @@ config = Config(
             working_dir="./rsync-workdir/",
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 launcher=SimpleLauncher(),

parsl/tests/configs/slurm_local.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
@@ -13,7 +12,6 @@ def fresh_config():
                 encrypted=True,
                 provider=SlurmProvider(
                     cmd_timeout=60,     # Add extra time for slow scheduler responses
-                    channel=LocalChannel(),
                     nodes_per_block=1,
                     init_blocks=1,
                     min_blocks=1,

parsl/tests/manual_tests/htex_local.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 
@@ -15,7 +14,6 @@ config = Config(
             cores_per_worker=1,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
                 # tasks_per_node=1,  # For HighThroughputExecutor, this option should in most cases be 1

parsl/tests/manual_tests/test_memory_limits.py

@@ -5,7 +5,6 @@ import psutil
 
 import parsl
 from parsl.app.app import python_app  # , bash_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SingleNodeLauncher
@@ -30,7 +29,6 @@ def test_simple(mem_per_worker):
                 suppress_failure=True,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                     launcher=SingleNodeLauncher(),

parsl/tests/scaling_tests/htex_local.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
@@ -12,7 +11,6 @@ config = Config(
             max_workers_per_node=8,
             encrypted=True,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
             ),

parsl/tests/sites/test_affinity.py

@@ -5,7 +5,6 @@ import os
 import pytest
 
 from parsl import python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
@@ -22,7 +21,6 @@ def local_config():
                 available_accelerators=2,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                 ),

parsl/tests/sites/test_worker_info.py

@@ -3,7 +3,6 @@
 import pytest
 
 from parsl import python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
@@ -18,7 +17,6 @@ def local_config():
                 max_workers_per_node=4,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                 ),

parsl/tests/test_htex/test_drain.py

@@ -3,7 +3,6 @@ import time
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -28,7 +27,6 @@ def local_config():
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     min_blocks=0,
                     max_blocks=0,

parsl/tests/test_htex/test_manager_selector_by_block.py

@@ -4,7 +4,6 @@ import pytest
 
 import parsl
 from parsl.app.app import bash_app, python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.executors.high_throughput.manager_selector import (
@@ -31,7 +30,6 @@ def test_block_id_selection(try_assert):
         max_workers_per_node=1,
         manager_selector=BlockIdManagerSelector(),
         provider=LocalProvider(
-            channel=LocalChannel(),
             init_blocks=BLOCK_COUNT,
             max_blocks=BLOCK_COUNT,
             min_blocks=BLOCK_COUNT,

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -5,7 +5,6 @@ import time
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -22,7 +21,6 @@ def fresh_config(run_dir, strategy, db_url):
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     # min and max are set to 0 to ensure that we don't get
                     # a block from ongoing strategy scaling, only from

parsl/tests/test_providers/test_local_provider.py

@@ -11,7 +11,6 @@ import time
 
 import pytest
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers import LocalProvider
@@ -63,7 +62,7 @@ def _run_tests(p: LocalProvider):
 def test_local_channel():
     with tempfile.TemporaryDirectory() as script_dir:
         script_dir = tempfile.mkdtemp()
-        p = LocalProvider(channel=LocalChannel(), launcher=SingleNodeLauncher(debug=False))
+        p = LocalProvider(launcher=SingleNodeLauncher(debug=False))
         p.script_dir = script_dir
         _run_tests(p)
 

parsl/tests/test_providers/test_pbspro_template.py

@@ -3,7 +3,6 @@ from unittest import mock
 
 import pytest
 
-from parsl.channels import LocalChannel
 from parsl.providers import PBSProProvider
 
 
@@ -12,10 +11,9 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = PBSProProvider(
-        queue="debug", channel=LocalChannel()
+        queue="debug"
     )
     provider.script_dir = tmp_path
-    provider.channel.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))
     provider.execute_wait = mock.Mock(spec=PBSProProvider.execute_wait)
     provider.execute_wait.return_value = (0, job_id, "")

parsl/tests/test_providers/test_slurm_template.py

@@ -4,7 +4,6 @@ from unittest import mock
 
 import pytest
 
-from parsl.channels import LocalChannel
 from parsl.providers import SlurmProvider
 
 
@@ -13,10 +12,9 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = SlurmProvider(
-        partition="debug", channel=LocalChannel()
+        partition="debug"
     )
     provider.script_dir = tmp_path
-    provider.channel.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))
     provider.execute_wait = mock.MagicMock(spec=SlurmProvider.execute_wait)
     provider.execute_wait.return_value = (0, f"Submitted batch job {job_id}", "")

parsl/tests/test_scaling/test_regression_1621.py

@@ -3,7 +3,6 @@ import threading
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -43,7 +42,6 @@ def test_one_block(tmpd_cwd):
     one app is invoked. this is a regression test.
     """
     oneshot_provider = OneShotLocalProvider(
-        channel=LocalChannel(),
         init_blocks=0,
         min_blocks=0,
         max_blocks=10,

parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py

@@ -3,7 +3,6 @@ import time
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import WrappedLauncher

parsl/tests/test_scaling/test_scale_down.py

@@ -5,7 +5,6 @@ import pytest
 
 import parsl
 from parsl import File, python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SingleNodeLauncher
@@ -29,7 +28,6 @@ def local_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -4,7 +4,6 @@ import pytest
 
 import parsl
 from parsl import File, python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SingleNodeLauncher
@@ -26,7 +25,6 @@ def local_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,

parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -5,7 +5,6 @@ import pytest
 
 import parsl
 from parsl import File, python_app
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.jobs.states import TERMINAL_STATES, JobState
@@ -31,7 +30,6 @@ def local_config():
                 encrypted=True,
                 launch_cmd="sleep inf",
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -4,7 +4,6 @@ import threading
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -47,7 +46,6 @@ def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
     scaled in at DFK shutdown.
     """
     accumulating_provider = AccumulatingLocalProvider(
-        channel=LocalChannel(),
         init_blocks=BLOCK_COUNT,
         min_blocks=0,
         max_blocks=0,

parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py

@@ -6,7 +6,6 @@ import pytest
 import zmq
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
@@ -24,7 +23,6 @@ def fresh_config():
                cores_per_worker=1,
                encrypted=False,
                provider=LocalProvider(
-                   channel=LocalChannel(),
                    init_blocks=0,
                    min_blocks=0,
                    max_blocks=0,

parsl/tests/test_staging/test_zip_in.py

@@ -4,7 +4,6 @@ import zipfile
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging

parsl/tests/test_staging/test_zip_out.py

@@ -3,7 +3,6 @@ import zipfile
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.data_manager import default_staging
 from parsl.data_provider.files import File

parsl/tests/test_staging/test_zip_to_zip.py

@@ -4,7 +4,6 @@ import zipfile
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging

parsl/tests/test_utils/test_execute_wait.py

@@ -0,0 +1,35 @@
+import pytest
+
+from parsl.utils import execute_wait
+
+
+@pytest.mark.local
+def test_env():
+    ''' Regression testing for issue #27
+    '''
+
+    rc, stdout, stderr = execute_wait("env", 1)
+
+    stdout = stdout.split('\n')
+    x = [s for s in stdout if s.startswith("PATH=")]
+    assert x, "PATH not found"
+
+    x = [s for s in stdout if s.startswith("HOME=")]
+    assert x, "HOME not found"
+
+
+@pytest.mark.local
+def test_large_output_2210():
+    """Regression test for #2210.
+    execute_wait was hanging if the specified command gave too
+    much output, due to a race condition between process exiting and
+    pipes filling up.
+    """
+
+    # this will output 128kb of stdout
+    execute_wait("yes | dd count=128 bs=1024", walltime=60)
+
+    # if this test fails, execute_wait should raise a timeout
+    # exception.
+
+    # The contents out the output is not verified by this test

parsl/utils.py

@@ -458,3 +458,38 @@ def sanitize_dns_subdomain_rfc1123(raw_string: str) -> str:
         raise ValueError(f"Sanitized DNS subdomain is empty for input '{raw_string}'")
 
     return sanitized
+
+
+def execute_wait(cmd: str, walltime: Optional[int] = None) -> Tuple[int, str, str]:
+    ''' Synchronously execute a commandline string on the shell.
+
+    Args:
+        - cmd (string) : Commandline string to execute
+        - walltime (int) : walltime in seconds
+
+    Returns:
+        - retcode : Return code from the execution
+        - stdout  : stdout string
+        - stderr  : stderr string
+    '''
+    try:
+        logger.debug("Creating process with command '%s'", cmd)
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=True,
+            preexec_fn=os.setpgrp
+        )
+        logger.debug("Created process with pid %s. Performing communicate", proc.pid)
+        (stdout, stderr) = proc.communicate(timeout=walltime)
+        retcode = proc.returncode
+        logger.debug("Process %s returned %s", proc.pid, proc.returncode)
+
+    except Exception:
+        logger.exception(f"Execution of command failed:\n{cmd}")
+        raise
+    else:
+        logger.debug("Execution of command in process %s completed normally", proc.pid)
+
+    return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.12.02'
+VERSION = '2024.12.16'

{parsl-2024.12.2.data → parsl-2024.12.16.data}/scripts/interchange.py

@@ -20,7 +20,8 @@ from parsl.executors.high_throughput.errors import ManagerLost, VersionMismatch
 from parsl.executors.high_throughput.manager_record import ManagerRecord
 from parsl.executors.high_throughput.manager_selector import ManagerSelector
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.radios import MonitoringRadioSender, ZMQRadioSender
+from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.zmq import ZMQRadioSender
 from parsl.process_loggers import wrap_with_logs
 from parsl.serialize import serialize as serialize_object
 from parsl.utils import setproctitle

{parsl-2024.12.2.dist-info → parsl-2024.12.16.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.12.2
+Version: 2024.12.16
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.12.02.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.12.16.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0