parsl 2024.11.4__py3-none-any.whl → 2024.11.18__py3-none-any.whl

parsl/channels/base.py

@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
-from typing import Dict, Tuple
+from typing import Tuple
 
 
 class Channel(metaclass=ABCMeta):
@@ -8,33 +8,27 @@ class Channel(metaclass=ABCMeta):
 
     For certain resources such as campus clusters or supercomputers at
     research laboratories, resource requirements may require authentication.
-    For instance some resources may allow access to their job schedulers from
-    only their login-nodes which require you to authenticate through SSH, or
-    require two factor authentication.
 
-    The simplest Channel, *LocalChannel*, executes commands locally in a
-    shell, while the *SSHChannel* authenticates you to remote systems.
+    The only remaining Channel, *LocalChannel*, executes commands locally in a
+    shell.
 
     Channels provide the ability to execute commands remotely, using the
     execute_wait method, and manipulate the remote file system using methods
     such as push_file, pull_file and makedirs.
 
     Channels should ensure that each launched command runs in a new process
-    group, so that providers (such as AdHocProvider and LocalProvider) which
-    terminate long running commands using process groups can do so.
+    group, so that providers (such as LocalProvider) which terminate long
+    running commands using process groups can do so.
     """
 
     @abstractmethod
-    def execute_wait(self, cmd: str, walltime: int = 0, envs: Dict[str, str] = {}) -> Tuple[int, str, str]:
+    def execute_wait(self, cmd: str, walltime: int = 0) -> Tuple[int, str, str]:
         ''' Executes the cmd, with a defined walltime.
 
         Args:
             - cmd (string): Command string to execute over the channel
             - walltime (int) : Timeout in seconds
 
-        KWargs:
-            - envs (Dict[str, str]) : Environment variables to push to the remote side
-
         Returns:
             - (exit_code, stdout, stderr) (int, string, string)
         '''
@@ -86,37 +80,3 @@ class Channel(metaclass=ABCMeta):
             destination_path (string)
         '''
         pass
-
-    @abstractmethod
-    def close(self) -> None:
-        ''' Closes the channel.
-        '''
-        pass
-
-    @abstractmethod
-    def makedirs(self, path: str, mode: int = 0o511, exist_ok: bool = False) -> None:
-        """Create a directory.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-        pass
-
-    @abstractmethod
-    def isdir(self, path: str) -> bool:
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to check.
-        """
-        pass

parsl/channels/errors.py

@@ -17,73 +17,6 @@ class ChannelError(ParslError):
         return "Hostname:{0}, Reason:{1}".format(self.hostname, self.reason)
 
 
-class BadHostKeyException(ChannelError):
-    ''' SSH channel could not be created since server's host keys could not
-    be verified
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("SSH channel could not be created since server's host keys could not be "
-                         "verified", e, hostname)
-
-
-class BadScriptPath(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Inaccessible remote script dir. Specify script_dir", e, hostname)
-
-
-class BadPermsScriptPath(ChannelError):
-    ''' User does not have permissions to access the script_dir on the remote site
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("User does not have permissions to access the script_dir", e, hostname)
-
-
-class AuthException(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Authentication to remote server failed", e, hostname)
-
-
-class SSHException(ChannelError):
-    ''' if there was any other error connecting or establishing an SSH session
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Error connecting or establishing an SSH session", e, hostname)
-
-
 class FileCopyException(ChannelError):
     ''' File copy operation failed
 

parsl/channels/local/local.py

@@ -1,4 +1,3 @@
-import copy
 import logging
 import os
 import shutil
@@ -16,49 +15,32 @@ class LocalChannel(Channel, RepresentationMixin):
     and done so infrequently that they do not need a persistent channel
     '''
 
-    def __init__(self, userhome=".", envs={}, script_dir=None):
+    def __init__(self):
         ''' Initialize the local channel. script_dir is required by set to a default.
 
         KwArgs:
-            - userhome (string): (default='.') This is provided as a way to override and set a specific userhome
-            - envs (dict) : A dictionary of env variables to be set when launching the shell
             - script_dir (string): Directory to place scripts
         '''
-        self.userhome = os.path.abspath(userhome)
-        self.hostname = "localhost"
-        self.envs = envs
-        local_env = os.environ.copy()
-        self._envs = copy.deepcopy(local_env)
-        self._envs.update(envs)
-        self.script_dir = script_dir
-
-    def execute_wait(self, cmd, walltime=None, envs={}):
+        self.script_dir = None
+
+    def execute_wait(self, cmd, walltime=None):
         ''' Synchronously execute a commandline string on the shell.
 
         Args:
             - cmd (string) : Commandline string to execute
             - walltime (int) : walltime in seconds
 
-        Kwargs:
-            - envs (dict) : Dictionary of env variables. This will be used
-              to override the envs set at channel initialization.
-
         Returns:
             - retcode : Return code from the execution
             - stdout  : stdout string
             - stderr  : stderr string
         '''
-        current_env = copy.deepcopy(self._envs)
-        current_env.update(envs)
-
         try:
             logger.debug("Creating process with command '%s'", cmd)
             proc = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                cwd=self.userhome,
-                env=current_env,
                 shell=True,
                 preexec_fn=os.setpgrp
             )
@@ -99,7 +81,7 @@ class LocalChannel(Channel, RepresentationMixin):
                 os.chmod(local_dest, 0o700)
 
             except OSError as e:
-                raise FileCopyException(e, self.hostname)
+                raise FileCopyException(e, "localhost")
 
         else:
             os.chmod(local_dest, 0o700)
@@ -109,39 +91,6 @@ class LocalChannel(Channel, RepresentationMixin):
     def pull_file(self, remote_source, local_dir):
         return self.push_file(remote_source, local_dir)
 
-    def close(self) -> None:
-        ''' There's nothing to close here, and so this doesn't do anything
-        '''
-        pass
-
-    def isdir(self, path):
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to check.
-        """
-
-        return os.path.isdir(path)
-
-    def makedirs(self, path, mode=0o700, exist_ok=False):
-        """Create a directory.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-
-        return os.makedirs(path, mode, exist_ok)
-
     @property
     def script_dir(self):
         return self._script_dir

parsl/dataflow/dflow.py

@@ -6,7 +6,6 @@ import datetime
 import inspect
 import logging
 import os
-import pathlib
 import pickle
 import random
 import sys
@@ -25,7 +24,6 @@ from typeguard import typechecked
 import parsl
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.app.futures import DataFuture
-from parsl.channels import Channel
 from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
@@ -49,7 +47,6 @@ from parsl.monitoring import MonitoringHub
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.remote import monitor_wrapper
 from parsl.process_loggers import wrap_with_logs
-from parsl.providers.base import ExecutionProvider
 from parsl.usage_tracking.usage import UsageTracker
 from parsl.utils import Timer, get_all_checkpoints, get_std_fname_mode, get_version
 
@@ -114,8 +111,6 @@ class DataFlowKernel:
         self.monitoring = config.monitoring
 
         if self.monitoring:
-            if self.monitoring.logdir is None:
-                self.monitoring.logdir = self.run_dir
             self.monitoring.start(self.run_dir, self.config.run_dir)
 
         self.time_began = datetime.datetime.now()
@@ -1143,36 +1138,6 @@ class DataFlowKernel:
 
         logger.info("End of summary")
 
-    def _create_remote_dirs_over_channel(self, provider: ExecutionProvider, channel: Channel) -> None:
-        """Create script directories across a channel
-
-        Parameters
-        ----------
-        provider: Provider obj
-           Provider for which scripts dirs are being created
-        channel: Channel obj
-           Channel over which the remote dirs are to be created
-        """
-        run_dir = self.run_dir
-        if channel.script_dir is None:
-
-            # This case will be detected as unreachable by mypy, because of
-            # the type of script_dir, which is str, not Optional[str].
-            # The type system doesn't represent the initialized/uninitialized
-            # state of a channel so cannot represent that a channel needs
-            # its script directory set or not.
-
-            channel.script_dir = os.path.join(run_dir, 'submit_scripts')  # type: ignore[unreachable]
-
-            # Only create dirs if we aren't on a shared-fs
-            if not channel.isdir(run_dir):
-                parent, child = pathlib.Path(run_dir).parts[-2:]
-                remote_run_dir = os.path.join(parent, child)
-                channel.script_dir = os.path.join(remote_run_dir, 'remote_submit_scripts')
-                provider.script_dir = os.path.join(run_dir, 'local_submit_scripts')
-
-        channel.makedirs(channel.script_dir, exist_ok=True)
-
     def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
             executor.run_id = self.run_id
@@ -1186,12 +1151,7 @@ class DataFlowKernel:
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')
                     os.makedirs(executor.provider.script_dir, exist_ok=True)
 
-                    if hasattr(executor.provider, 'channels'):
-                        logger.debug("Creating script_dir across multiple channels")
-                        for channel in executor.provider.channels:
-                            self._create_remote_dirs_over_channel(executor.provider, channel)
-                    else:
-                        self._create_remote_dirs_over_channel(executor.provider, executor.provider.channel)
+                    executor.provider.channel.script_dir = executor.provider.script_dir
 
             self.executors[executor.label] = executor
             executor.start()
@@ -1273,23 +1233,6 @@ class DataFlowKernel:
             executor.shutdown()
             logger.info(f"Shut down executor {executor.label}")
 
-            if hasattr(executor, 'provider'):
-                if hasattr(executor.provider, 'script_dir'):
-                    logger.info(f"Closing channel(s) for {executor.label}")
-
-                    if hasattr(executor.provider, 'channels'):
-                        for channel in executor.provider.channels:
-                            logger.info(f"Closing channel {channel}")
-                            channel.close()
-                            logger.info(f"Closed channel {channel}")
-                    else:
-                        assert hasattr(executor.provider, 'channel'), "If provider has no .channels, it must have .channel"
-                        logger.info(f"Closing channel {executor.provider.channel}")
-                        executor.provider.channel.close()
-                        logger.info(f"Closed channel {executor.provider.channel}")
-
-                    logger.info(f"Closed executor channel(s) for {executor.label}")
-
         logger.info("Terminated executors")
         self.time_completed = datetime.datetime.now()
 

parsl/executors/taskvine/manager.py

@@ -44,11 +44,17 @@ def _set_manager_attributes(m, config):
     # Enable peer transfer feature between workers if specified
     if config.enable_peer_transfers:
         m.enable_peer_transfers()
+    else:
+        m.disable_peer_transfers()
 
     # Set catalog report to parsl if project name exists
     if m.name:
         m.set_property("framework", "parsl")
 
+    if config.tune_parameters is not None:
+        for k, v in config.tune_parameters.items():
+            m.tune(k, v)
+
 
 def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script):
     # Return path to a packaged poncho environment

parsl/executors/taskvine/manager_config.py

@@ -156,6 +156,10 @@ class TaskVineManagerConfig:
         Directory to store TaskVine logging facilities.
         Default is None, in which all TaskVine logs will be contained
         in the Parsl logging directory.
+
+    tune_parameters: Optional[dict]
+        Extended vine_tune parameters, expressed in a dictionary
+        by { 'tune-parameter' : value }.
     """
 
     # Connection and communication settings
@@ -181,6 +185,7 @@ class TaskVineManagerConfig:
     autocategory: bool = True
     enable_peer_transfers: bool = True
     wait_for_workers: Optional[int] = None
+    tune_parameters: Optional[dict] = None
 
     # Logging settings
     vine_log_dir: Optional[str] = None

parsl/monitoring/db_manager.py

@@ -279,7 +279,7 @@ class Database:
 class DatabaseManager:
     def __init__(self,
                  db_url: str = 'sqlite:///runinfo/monitoring.db',
-                 logdir: str = '.',
+                 run_dir: str = '.',
                  logging_level: int = logging.INFO,
                  batching_interval: float = 1,
                  batching_threshold: float = 99999,
@@ -287,12 +287,12 @@ class DatabaseManager:
 
         self.workflow_end = False
         self.workflow_start_message: Optional[MonitoringMessage] = None
-        self.logdir = logdir
-        os.makedirs(self.logdir, exist_ok=True)
+        self.run_dir = run_dir
+        os.makedirs(self.run_dir, exist_ok=True)
 
         logger.propagate = False
 
-        set_file_logger("{}/database_manager.log".format(self.logdir), level=logging_level,
+        set_file_logger(f"{self.run_dir}/database_manager.log", level=logging_level,
                         format_string="%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s] [%(threadName)s %(thread)d] %(message)s",
                         name="database_manager")
 
@@ -681,7 +681,7 @@ class DatabaseManager:
 def dbm_starter(exception_q: mpq.Queue,
                 resource_msgs: mpq.Queue,
                 db_url: str,
-                logdir: str,
+                run_dir: str,
                 logging_level: int) -> None:
     """Start the database manager process
 
@@ -692,7 +692,7 @@ def dbm_starter(exception_q: mpq.Queue,
 
     try:
         dbm = DatabaseManager(db_url=db_url,
-                              logdir=logdir,
+                              run_dir=run_dir,
                               logging_level=logging_level)
         logger.info("Starting dbm in dbm starter")
         dbm.start(resource_msgs)

parsl/monitoring/monitoring.py

@@ -3,9 +3,10 @@ from __future__ import annotations
 import logging
 import multiprocessing.synchronize as ms
 import os
+import pickle
 import queue
 import time
-from multiprocessing import Event, Process
+from multiprocessing import Event
 from multiprocessing.queues import Queue
 from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union, cast
 
@@ -18,7 +19,6 @@ from parsl.monitoring.router import router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue
 from parsl.process_loggers import wrap_with_logs
-from parsl.serialize import deserialize
 from parsl.utils import RepresentationMixin, setproctitle
 
 _db_manager_excepts: Optional[Exception]
@@ -44,7 +44,6 @@ class MonitoringHub(RepresentationMixin):
                  workflow_name: Optional[str] = None,
                  workflow_version: Optional[str] = None,
                  logging_endpoint: Optional[str] = None,
-                 logdir: Optional[str] = None,
                  monitoring_debug: bool = False,
                  resource_monitoring_enabled: bool = True,
                  resource_monitoring_interval: float = 30):  # in seconds
@@ -73,8 +72,6 @@ class MonitoringHub(RepresentationMixin):
              The database connection url for monitoring to log the information.
              These URLs follow RFC-1738, and can include username, password, hostname, database name.
              Default: sqlite, in the configured run_dir.
-        logdir : str
-             Parsl log directory paths. Logs and temp files go here. Default: '.'
         monitoring_debug : Bool
              Enable monitoring debug logging. Default: False
         resource_monitoring_enabled : boolean
@@ -96,7 +93,6 @@ class MonitoringHub(RepresentationMixin):
         self.hub_port_range = hub_port_range
 
         self.logging_endpoint = logging_endpoint
-        self.logdir = logdir
         self.monitoring_debug = monitoring_debug
 
         self.workflow_name = workflow_name
@@ -109,13 +105,10 @@ class MonitoringHub(RepresentationMixin):
 
         logger.debug("Starting MonitoringHub")
 
-        if self.logdir is None:
-            self.logdir = "."
-
         if self.logging_endpoint is None:
             self.logging_endpoint = f"sqlite:///{os.fspath(config_run_dir)}/monitoring.db"
 
-        os.makedirs(self.logdir, exist_ok=True)
+        os.makedirs(dfk_run_dir, exist_ok=True)
 
         self.monitoring_hub_active = True
 
@@ -151,7 +144,7 @@ class MonitoringHub(RepresentationMixin):
                                                "hub_address": self.hub_address,
                                                "udp_port": self.hub_port,
                                                "zmq_port_range": self.hub_port_range,
-                                               "logdir": self.logdir,
+                                               "run_dir": dfk_run_dir,
                                                "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                                },
                                        name="Monitoring-Router-Process",
@@ -161,7 +154,7 @@ class MonitoringHub(RepresentationMixin):
 
         self.dbm_proc = ForkProcess(target=dbm_starter,
                                     args=(self.exception_q, self.resource_msgs,),
-                                    kwargs={"logdir": self.logdir,
+                                    kwargs={"run_dir": dfk_run_dir,
                                             "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                             "db_url": self.logging_endpoint,
                                             },
@@ -169,15 +162,15 @@ class MonitoringHub(RepresentationMixin):
                                     daemon=True,
                                     )
         self.dbm_proc.start()
-        logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
+        logger.info("Started the router process %s and DBM process %s", self.router_proc.pid, self.dbm_proc.pid)
 
-        self.filesystem_proc = Process(target=filesystem_receiver,
-                                       args=(self.logdir, self.resource_msgs, dfk_run_dir),
-                                       name="Monitoring-Filesystem-Process",
-                                       daemon=True
-                                       )
+        self.filesystem_proc = ForkProcess(target=filesystem_receiver,
+                                           args=(self.resource_msgs, dfk_run_dir),
+                                           name="Monitoring-Filesystem-Process",
+                                           daemon=True
+                                           )
         self.filesystem_proc.start()
-        logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
+        logger.info("Started filesystem radio receiver process %s", self.filesystem_proc.pid)
 
         self.radio = MultiprocessingQueueRadioSender(self.resource_msgs)
 
@@ -190,7 +183,7 @@ class MonitoringHub(RepresentationMixin):
             raise MonitoringHubStartError()
 
         if isinstance(comm_q_result, str):
-            logger.error(f"MonitoringRouter sent an error message: {comm_q_result}")
+            logger.error("MonitoringRouter sent an error message: %s", comm_q_result)
             raise RuntimeError(f"MonitoringRouter failed to start: {comm_q_result}")
 
         udp_port, zmq_port = comm_q_result
@@ -202,7 +195,7 @@ class MonitoringHub(RepresentationMixin):
         self.hub_zmq_port = zmq_port
 
     def send(self, message: TaggedMonitoringMessage) -> None:
-        logger.debug("Sending message type {}".format(message[0]))
+        logger.debug("Sending message type %s", message[0])
         self.radio.send(message)
 
     def close(self) -> None:
@@ -219,10 +212,9 @@ class MonitoringHub(RepresentationMixin):
             if exception_msgs:
                 for exception_msg in exception_msgs:
                     logger.error(
-                        "{} process delivered an exception: {}. Terminating all monitoring processes immediately.".format(
-                            exception_msg[0],
-                            exception_msg[1]
-                        )
+                        "%s process delivered an exception: %s. Terminating all monitoring processes immediately.",
+                        exception_msg[0],
+                        exception_msg[1]
                     )
                 self.router_proc.terminate()
                 self.dbm_proc.terminate()
@@ -259,8 +251,8 @@ class MonitoringHub(RepresentationMixin):
 
 
 @wrap_with_logs
-def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]", run_dir: str) -> None:
-    logger = set_file_logger("{}/monitoring_filesystem_radio.log".format(logdir),
+def filesystem_receiver(q: Queue[TaggedMonitoringMessage], run_dir: str) -> None:
+    logger = set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
                              name="monitoring_filesystem_radio",
                              level=logging.INFO)
 
@@ -269,7 +261,9 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]",
     base_path = f"{run_dir}/monitor-fs-radio/"
     tmp_dir = f"{base_path}/tmp/"
     new_dir = f"{base_path}/new/"
-    logger.debug(f"Creating new and tmp paths under {base_path}")
+    logger.debug("Creating new and tmp paths under %s", base_path)
+
+    target_radio = MultiprocessingQueueRadioSender(q)
 
     os.makedirs(tmp_dir, exist_ok=True)
     os.makedirs(new_dir, exist_ok=True)
@@ -280,15 +274,15 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[TaggedMonitoringMessage]",
         # iterate over files in new_dir
         for filename in os.listdir(new_dir):
             try:
-                logger.info(f"Processing filesystem radio file {filename}")
+                logger.info("Processing filesystem radio file %s", filename)
                 full_path_filename = f"{new_dir}/{filename}"
                 with open(full_path_filename, "rb") as f:
-                    message = deserialize(f.read())
-                logger.debug(f"Message received is: {message}")
+                    message = pickle.load(f)
+                logger.debug("Message received is: %s", message)
                 assert isinstance(message, tuple)
-                q.put(cast(TaggedMonitoringMessage, message))
+                target_radio.send(cast(TaggedMonitoringMessage, message))
                 os.remove(full_path_filename)
             except Exception:
-                logger.exception(f"Exception processing {filename} - probably will be retried next iteration")
+                logger.exception("Exception processing %s - probably will be retried next iteration", filename)
 
         time.sleep(1)  # whats a good time for this poll?

parsl/monitoring/radios.py

@@ -8,8 +8,6 @@ from multiprocessing.queues import Queue
 
 import zmq
 
-from parsl.serialize import serialize
-
 logger = logging.getLogger(__name__)
 
 
@@ -59,7 +57,7 @@ class FilesystemRadioSender(MonitoringRadioSender):
         # move it into new/, so that a partially written
         # file will never be observed in new/
         with open(tmp_filename, "wb") as f:
-            f.write(serialize(buffer))
+            pickle.dump(buffer, f)
         os.rename(tmp_filename, new_filename)