parsl 2024.10.28__py3-none-any.whl → 2024.11.11__py3-none-any.whl

parsl/channels/base.py

@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
-from typing import Dict, Tuple
+from typing import Tuple
 
 
 class Channel(metaclass=ABCMeta):
@@ -8,33 +8,27 @@ class Channel(metaclass=ABCMeta):
 
     For certain resources such as campus clusters or supercomputers at
     research laboratories, resource requirements may require authentication.
-    For instance some resources may allow access to their job schedulers from
-    only their login-nodes which require you to authenticate through SSH, or
-    require two factor authentication.
 
-    The simplest Channel, *LocalChannel*, executes commands locally in a
-    shell, while the *SSHChannel* authenticates you to remote systems.
+    The only remaining Channel, *LocalChannel*, executes commands locally in a
+    shell.
 
     Channels provide the ability to execute commands remotely, using the
     execute_wait method, and manipulate the remote file system using methods
     such as push_file, pull_file and makedirs.
 
     Channels should ensure that each launched command runs in a new process
-    group, so that providers (such as AdHocProvider and LocalProvider) which
-    terminate long running commands using process groups can do so.
+    group, so that providers (such as LocalProvider) which terminate long
+    running commands using process groups can do so.
     """
 
     @abstractmethod
-    def execute_wait(self, cmd: str, walltime: int = 0, envs: Dict[str, str] = {}) -> Tuple[int, str, str]:
+    def execute_wait(self, cmd: str, walltime: int = 0) -> Tuple[int, str, str]:
         ''' Executes the cmd, with a defined walltime.
 
         Args:
             - cmd (string): Command string to execute over the channel
             - walltime (int) : Timeout in seconds
 
-        KWargs:
-            - envs (Dict[str, str]) : Environment variables to push to the remote side
-
         Returns:
             - (exit_code, stdout, stderr) (int, string, string)
         '''
@@ -86,37 +80,3 @@ class Channel(metaclass=ABCMeta):
             destination_path (string)
         '''
         pass
-
-    @abstractmethod
-    def close(self) -> None:
-        ''' Closes the channel.
-        '''
-        pass
-
-    @abstractmethod
-    def makedirs(self, path: str, mode: int = 0o511, exist_ok: bool = False) -> None:
-        """Create a directory.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-        pass
-
-    @abstractmethod
-    def isdir(self, path: str) -> bool:
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to check.
-        """
-        pass

parsl/channels/errors.py

@@ -17,73 +17,6 @@ class ChannelError(ParslError):
         return "Hostname:{0}, Reason:{1}".format(self.hostname, self.reason)
 
 
-class BadHostKeyException(ChannelError):
-    ''' SSH channel could not be created since server's host keys could not
-    be verified
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("SSH channel could not be created since server's host keys could not be "
-                         "verified", e, hostname)
-
-
-class BadScriptPath(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Inaccessible remote script dir. Specify script_dir", e, hostname)
-
-
-class BadPermsScriptPath(ChannelError):
-    ''' User does not have permissions to access the script_dir on the remote site
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("User does not have permissions to access the script_dir", e, hostname)
-
-
-class AuthException(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Authentication to remote server failed", e, hostname)
-
-
-class SSHException(ChannelError):
-    ''' if there was any other error connecting or establishing an SSH session
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Error connecting or establishing an SSH session", e, hostname)
-
-
 class FileCopyException(ChannelError):
     ''' File copy operation failed
 

parsl/channels/local/local.py

@@ -1,4 +1,3 @@
-import copy
 import logging
 import os
 import shutil
@@ -16,49 +15,32 @@ class LocalChannel(Channel, RepresentationMixin):
     and done so infrequently that they do not need a persistent channel
     '''
 
-    def __init__(self, userhome=".", envs={}, script_dir=None):
+    def __init__(self):
         ''' Initialize the local channel. script_dir is required by set to a default.
 
         KwArgs:
-            - userhome (string): (default='.') This is provided as a way to override and set a specific userhome
-            - envs (dict) : A dictionary of env variables to be set when launching the shell
             - script_dir (string): Directory to place scripts
         '''
-        self.userhome = os.path.abspath(userhome)
-        self.hostname = "localhost"
-        self.envs = envs
-        local_env = os.environ.copy()
-        self._envs = copy.deepcopy(local_env)
-        self._envs.update(envs)
-        self.script_dir = script_dir
-
-    def execute_wait(self, cmd, walltime=None, envs={}):
+        self.script_dir = None
+
+    def execute_wait(self, cmd, walltime=None):
         ''' Synchronously execute a commandline string on the shell.
 
         Args:
             - cmd (string) : Commandline string to execute
             - walltime (int) : walltime in seconds
 
-        Kwargs:
-            - envs (dict) : Dictionary of env variables. This will be used
-              to override the envs set at channel initialization.
-
         Returns:
             - retcode : Return code from the execution
             - stdout  : stdout string
             - stderr  : stderr string
         '''
-        current_env = copy.deepcopy(self._envs)
-        current_env.update(envs)
-
         try:
             logger.debug("Creating process with command '%s'", cmd)
             proc = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                cwd=self.userhome,
-                env=current_env,
                 shell=True,
                 preexec_fn=os.setpgrp
             )
@@ -99,7 +81,7 @@ class LocalChannel(Channel, RepresentationMixin):
                 os.chmod(local_dest, 0o700)
 
             except OSError as e:
-                raise FileCopyException(e, self.hostname)
+                raise FileCopyException(e, "localhost")
 
         else:
             os.chmod(local_dest, 0o700)
@@ -109,39 +91,6 @@ class LocalChannel(Channel, RepresentationMixin):
     def pull_file(self, remote_source, local_dir):
         return self.push_file(remote_source, local_dir)
 
-    def close(self) -> None:
-        ''' There's nothing to close here, and so this doesn't do anything
-        '''
-        pass
-
-    def isdir(self, path):
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to check.
-        """
-
-        return os.path.isdir(path)
-
-    def makedirs(self, path, mode=0o700, exist_ok=False):
-        """Create a directory.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-
-        return os.makedirs(path, mode, exist_ok)
-
     @property
     def script_dir(self):
         return self._script_dir

parsl/dataflow/dflow.py

@@ -6,7 +6,6 @@ import datetime
 import inspect
 import logging
 import os
-import pathlib
 import pickle
 import random
 import sys
@@ -25,7 +24,6 @@ from typeguard import typechecked
 import parsl
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.app.futures import DataFuture
-from parsl.channels import Channel
 from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
@@ -49,7 +47,6 @@ from parsl.monitoring import MonitoringHub
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.remote import monitor_wrapper
 from parsl.process_loggers import wrap_with_logs
-from parsl.providers.base import ExecutionProvider
 from parsl.usage_tracking.usage import UsageTracker
 from parsl.utils import Timer, get_all_checkpoints, get_std_fname_mode, get_version
 
@@ -162,8 +159,8 @@ class DataFlowKernel:
         }
 
         if self.monitoring:
-            self.monitoring.send(MessageType.WORKFLOW_INFO,
-                                 workflow_info)
+            self.monitoring.send((MessageType.WORKFLOW_INFO,
+                                 workflow_info))
 
         if config.checkpoint_files is not None:
             checkpoints = self.load_checkpoints(config.checkpoint_files)
@@ -238,7 +235,7 @@ class DataFlowKernel:
     def _send_task_log_info(self, task_record: TaskRecord) -> None:
         if self.monitoring:
             task_log_info = self._create_task_log_info(task_record)
-            self.monitoring.send(MessageType.TASK_INFO, task_log_info)
+            self.monitoring.send((MessageType.TASK_INFO, task_log_info))
 
     def _create_task_log_info(self, task_record: TaskRecord) -> Dict[str, Any]:
         """
@@ -1143,36 +1140,6 @@ class DataFlowKernel:
 
         logger.info("End of summary")
 
-    def _create_remote_dirs_over_channel(self, provider: ExecutionProvider, channel: Channel) -> None:
-        """Create script directories across a channel
-
-        Parameters
-        ----------
-        provider: Provider obj
-           Provider for which scripts dirs are being created
-        channel: Channel obj
-           Channel over which the remote dirs are to be created
-        """
-        run_dir = self.run_dir
-        if channel.script_dir is None:
-
-            # This case will be detected as unreachable by mypy, because of
-            # the type of script_dir, which is str, not Optional[str].
-            # The type system doesn't represent the initialized/uninitialized
-            # state of a channel so cannot represent that a channel needs
-            # its script directory set or not.
-
-            channel.script_dir = os.path.join(run_dir, 'submit_scripts')  # type: ignore[unreachable]
-
-            # Only create dirs if we aren't on a shared-fs
-            if not channel.isdir(run_dir):
-                parent, child = pathlib.Path(run_dir).parts[-2:]
-                remote_run_dir = os.path.join(parent, child)
-                channel.script_dir = os.path.join(remote_run_dir, 'remote_submit_scripts')
-                provider.script_dir = os.path.join(run_dir, 'local_submit_scripts')
-
-        channel.makedirs(channel.script_dir, exist_ok=True)
-
     def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
             executor.run_id = self.run_id
@@ -1186,12 +1153,7 @@ class DataFlowKernel:
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')
                     os.makedirs(executor.provider.script_dir, exist_ok=True)
 
-                    if hasattr(executor.provider, 'channels'):
-                        logger.debug("Creating script_dir across multiple channels")
-                        for channel in executor.provider.channels:
-                            self._create_remote_dirs_over_channel(executor.provider, channel)
-                    else:
-                        self._create_remote_dirs_over_channel(executor.provider, executor.provider.channel)
+                    executor.provider.channel.script_dir = executor.provider.script_dir
 
             self.executors[executor.label] = executor
             executor.start()
@@ -1273,34 +1235,17 @@ class DataFlowKernel:
             executor.shutdown()
             logger.info(f"Shut down executor {executor.label}")
 
-            if hasattr(executor, 'provider'):
-                if hasattr(executor.provider, 'script_dir'):
-                    logger.info(f"Closing channel(s) for {executor.label}")
-
-                    if hasattr(executor.provider, 'channels'):
-                        for channel in executor.provider.channels:
-                            logger.info(f"Closing channel {channel}")
-                            channel.close()
-                            logger.info(f"Closed channel {channel}")
-                    else:
-                        assert hasattr(executor.provider, 'channel'), "If provider has no .channels, it must have .channel"
-                        logger.info(f"Closing channel {executor.provider.channel}")
-                        executor.provider.channel.close()
-                        logger.info(f"Closed channel {executor.provider.channel}")
-
-                    logger.info(f"Closed executor channel(s) for {executor.label}")
-
         logger.info("Terminated executors")
         self.time_completed = datetime.datetime.now()
 
         if self.monitoring:
             logger.info("Sending final monitoring message")
-            self.monitoring.send(MessageType.WORKFLOW_INFO,
+            self.monitoring.send((MessageType.WORKFLOW_INFO,
                                  {'tasks_failed_count': self.task_state_counts[States.failed],
                                   'tasks_completed_count': self.task_state_counts[States.exec_done],
                                   "time_began": self.time_began,
                                   'time_completed': self.time_completed,
-                                  'run_id': self.run_id, 'rundir': self.run_dir})
+                                  'run_id': self.run_id, 'rundir': self.run_dir}))
 
             logger.info("Terminating monitoring")
             self.monitoring.close()

parsl/executors/high_throughput/executor.py

@@ -63,7 +63,6 @@ DEFAULT_INTERCHANGE_LAUNCH_CMD = ["interchange.py"]
 
 GENERAL_HTEX_PARAM_DOCS = """provider : :class:`~parsl.providers.base.ExecutionProvider`
        Provider to access computation resources. Can be one of :class:`~parsl.providers.aws.aws.EC2Provider`,
-        :class:`~parsl.providers.cobalt.cobalt.Cobalt`,
         :class:`~parsl.providers.condor.condor.Condor`,
         :class:`~parsl.providers.googlecloud.googlecloud.GoogleCloud`,
         :class:`~parsl.providers.gridEngine.gridEngine.GridEngine`,

parsl/executors/high_throughput/mpi_resource_management.py

@@ -17,7 +17,6 @@ class Scheduler(Enum):
     Unknown = 0
     Slurm = 1
     PBS = 2
-    Cobalt = 3
 
 
 def get_slurm_hosts_list() -> List[str]:
@@ -37,13 +36,6 @@ def get_pbs_hosts_list() -> List[str]:
         return [line.strip() for line in f.readlines()]
 
 
-def get_cobalt_hosts_list() -> List[str]:
-    """Get list of COBALT hosts from envvar: COBALT_NODEFILE"""
-    nodefile_name = os.environ["COBALT_NODEFILE"]
-    with open(nodefile_name) as f:
-        return [line.strip() for line in f.readlines()]
-
-
 def get_nodes_in_batchjob(scheduler: Scheduler) -> List[str]:
     """Get nodelist from all supported schedulers"""
     nodelist = []
@@ -51,8 +43,6 @@ def get_nodes_in_batchjob(scheduler: Scheduler) -> List[str]:
         nodelist = get_slurm_hosts_list()
     elif scheduler == Scheduler.PBS:
         nodelist = get_pbs_hosts_list()
-    elif scheduler == Scheduler.Cobalt:
-        nodelist = get_cobalt_hosts_list()
     else:
         raise RuntimeError(f"mpi_mode does not support scheduler:{scheduler}")
     return nodelist
@@ -64,8 +54,6 @@ def identify_scheduler() -> Scheduler:
         return Scheduler.Slurm
     elif os.environ.get("PBS_NODEFILE"):
         return Scheduler.PBS
-    elif os.environ.get("COBALT_NODEFILE"):
-        return Scheduler.Cobalt
     else:
         return Scheduler.Unknown
 

parsl/executors/taskvine/manager.py

@@ -44,11 +44,17 @@ def _set_manager_attributes(m, config):
     # Enable peer transfer feature between workers if specified
     if config.enable_peer_transfers:
         m.enable_peer_transfers()
+    else:
+        m.disable_peer_transfers()
 
     # Set catalog report to parsl if project name exists
     if m.name:
         m.set_property("framework", "parsl")
 
+    if config.tune_parameters is not None:
+        for k, v in config.tune_parameters.items():
+            m.tune(k, v)
+
 
 def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script):
     # Return path to a packaged poncho environment

parsl/executors/taskvine/manager_config.py

@@ -156,6 +156,10 @@ class TaskVineManagerConfig:
         Directory to store TaskVine logging facilities.
         Default is None, in which all TaskVine logs will be contained
         in the Parsl logging directory.
+
+    tune_parameters: Optional[dict]
+        Extended vine_tune parameters, expressed in a dictionary
+        by { 'tune-parameter' : value }.
     """
 
     # Connection and communication settings
@@ -181,6 +185,7 @@ class TaskVineManagerConfig:
     autocategory: bool = True
     enable_peer_transfers: bool = True
     wait_for_workers: Optional[int] = None
+    tune_parameters: Optional[dict] = None
 
     # Logging settings
     vine_log_dir: Optional[str] = None

parsl/monitoring/monitoring.py

@@ -3,23 +3,22 @@ from __future__ import annotations
 import logging
 import multiprocessing.synchronize as ms
 import os
+import pickle
 import queue
 import time
-from multiprocessing import Event, Process
+from multiprocessing import Event
 from multiprocessing.queues import Queue
-from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union, cast
 
 import typeguard
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringHubStartError
-from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.radios import MultiprocessingQueueRadioSender
 from parsl.monitoring.router import router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue
 from parsl.process_loggers import wrap_with_logs
-from parsl.serialize import deserialize
 from parsl.utils import RepresentationMixin, setproctitle
 
 _db_manager_excepts: Optional[Exception]
@@ -170,15 +169,15 @@ class MonitoringHub(RepresentationMixin):
                                     daemon=True,
                                     )
         self.dbm_proc.start()
-        logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
+        logger.info("Started the router process %s and DBM process %s", self.router_proc.pid, self.dbm_proc.pid)
 
-        self.filesystem_proc = Process(target=filesystem_receiver,
-                                       args=(self.logdir, self.resource_msgs, dfk_run_dir),
-                                       name="Monitoring-Filesystem-Process",
-                                       daemon=True
-                                       )
+        self.filesystem_proc = ForkProcess(target=filesystem_receiver,
+                                           args=(self.logdir, self.resource_msgs, dfk_run_dir),
+                                           name="Monitoring-Filesystem-Process",
+                                           daemon=True
+                                           )
         self.filesystem_proc.start()
-        logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
+        logger.info("Started filesystem radio receiver process %s", self.filesystem_proc.pid)
 
         self.radio = MultiprocessingQueueRadioSender(self.resource_msgs)
 
@@ -191,7 +190,7 @@ class MonitoringHub(RepresentationMixin):
             raise MonitoringHubStartError()
 
         if isinstance(comm_q_result, str):
-            logger.error(f"MonitoringRouter sent an error message: {comm_q_result}")
+            logger.error("MonitoringRouter sent an error message: %s", comm_q_result)
             raise RuntimeError(f"MonitoringRouter failed to start: {comm_q_result}")
 
         udp_port, zmq_port = comm_q_result
@@ -202,10 +201,9 @@ class MonitoringHub(RepresentationMixin):
 
         self.hub_zmq_port = zmq_port
 
-    # TODO: tighten the Any message format
-    def send(self, mtype: MessageType, message: Any) -> None:
-        logger.debug("Sending message type {}".format(mtype))
-        self.radio.send((mtype, message))
+    def send(self, message: TaggedMonitoringMessage) -> None:
+        logger.debug("Sending message type %s", message[0])
+        self.radio.send(message)
 
     def close(self) -> None:
         logger.info("Terminating Monitoring Hub")
@@ -221,10 +219,9 @@ class MonitoringHub(RepresentationMixin):
             if exception_msgs:
                 for exception_msg in exception_msgs:
                     logger.error(
-                        "{} process delivered an exception: {}. Terminating all monitoring processes immediately.".format(
-                            exception_msg[0],
-                            exception_msg[1]
-                        )
+                        "%s process delivered an exception: %s. Terminating all monitoring processes immediately.",
+                        exception_msg[0],
+                        exception_msg[1]
                     )
                 self.router_proc.terminate()
                 self.dbm_proc.terminate()
@@ -261,7 +258,7 @@ class MonitoringHub(RepresentationMixin):
 
 
 @wrap_with_logs
-def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]", run_dir: str) -> None:
+def filesystem_receiver(logdir: str, q: Queue[TaggedMonitoringMessage], run_dir: str) -> None:
     logger = set_file_logger("{}/monitoring_filesystem_radio.log".format(logdir),
                              name="monitoring_filesystem_radio",
                              level=logging.INFO)
@@ -271,7 +268,7 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]",
     base_path = f"{run_dir}/monitor-fs-radio/"
     tmp_dir = f"{base_path}/tmp/"
     new_dir = f"{base_path}/new/"
-    logger.debug(f"Creating new and tmp paths under {base_path}")
+    logger.debug("Creating new and tmp paths under %s", base_path)
 
     os.makedirs(tmp_dir, exist_ok=True)
     os.makedirs(new_dir, exist_ok=True)
@@ -282,15 +279,15 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]",
         # iterate over files in new_dir
         for filename in os.listdir(new_dir):
             try:
-                logger.info(f"Processing filesystem radio file {filename}")
+                logger.info("Processing filesystem radio file %s", filename)
                 full_path_filename = f"{new_dir}/{filename}"
                 with open(full_path_filename, "rb") as f:
-                    message = deserialize(f.read())
-                logger.debug(f"Message received is: {message}")
+                    message = pickle.load(f)
+                logger.debug("Message received is: %s", message)
                 assert isinstance(message, tuple)
                 q.put(cast(TaggedMonitoringMessage, message))
                 os.remove(full_path_filename)
             except Exception:
-                logger.exception(f"Exception processing {filename} - probably will be retried next iteration")
+                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
 
         time.sleep(1)  # whats a good time for this poll?