parsl 2024.12.2__py3-none-any.whl → 2024.12.16__py3-none-any.whl

parsl/configs/cc_in2p3.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
@@ -10,7 +9,6 @@ config = Config(
             label='cc_in2p3_htex',
             max_workers_per_node=2,
             provider=GridEngineProvider(
-                channel=LocalChannel(),
                 nodes_per_block=1,
                 init_blocks=2,
                 max_blocks=2,

parsl/configs/frontera.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
@@ -15,7 +14,6 @@ config = Config(
             max_workers_per_node=1,          # Set number of workers per node
             provider=SlurmProvider(
                 cmd_timeout=60,     # Add extra time for slow scheduler responses
-                channel=LocalChannel(),
                 nodes_per_block=2,
                 init_blocks=1,
                 min_blocks=1,

parsl/configs/htex_local.py

@@ -1,4 +1,3 @@
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
@@ -10,7 +9,6 @@ config = Config(
             label="htex_local",
             cores_per_worker=1,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
             ),

parsl/dataflow/dflow.py

@@ -1151,8 +1151,6 @@ class DataFlowKernel:
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')
                     os.makedirs(executor.provider.script_dir, exist_ok=True)
 
-                    executor.provider.channel.script_dir = executor.provider.script_dir
-
             self.executors[executor.label] = executor
             executor.start()
         block_executors = [e for e in executors if isinstance(e, BlockProviderExecutor)]

parsl/executors/base.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Optional
 
 from typing_extensions import Literal, Self
 
-from parsl.monitoring.radios import MonitoringRadioSender
+from parsl.monitoring.radios.base import MonitoringRadioSender
 
 
 class ParslExecutor(metaclass=ABCMeta):

parsl/executors/high_throughput/interchange.py

@@ -20,7 +20,8 @@ from parsl.executors.high_throughput.errors import ManagerLost, VersionMismatch
 from parsl.executors.high_throughput.manager_record import ManagerRecord
 from parsl.executors.high_throughput.manager_selector import ManagerSelector
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.radios import MonitoringRadioSender, ZMQRadioSender
+from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.zmq import ZMQRadioSender
 from parsl.process_loggers import wrap_with_logs
 from parsl.serialize import serialize as serialize_object
 from parsl.utils import setproctitle

parsl/executors/radical/executor.py

@@ -133,6 +133,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
         self.resource = resource
         self._uid = RPEX.lower()
         self.bulk_mode = bulk_mode
+        self._terminate = mt.Event()
         self.working_dir = working_dir
         self.pilot_kwargs = rpex_pilot_kwargs
         self.future_tasks: Dict[str, Future] = {}
@@ -532,7 +533,7 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
 
         bulk = list()
 
-        while True:
+        while not self._terminate.is_set():
 
             now = time.time()  # time of last submission
 
@@ -552,6 +553,9 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
                 if len(bulk) >= self._max_bulk_size:
                     break
 
+                if self._terminate.is_set():
+                    break
+
             if bulk:
                 logger.debug('submit bulk: %d', len(bulk))
                 self.tmgr.submit_tasks(bulk)
@@ -588,6 +592,13 @@ class RadicalPilotExecutor(ParslExecutor, RepresentationMixin):
     def shutdown(self, hub=True, targets='all', block=False):
         """Shutdown the executor, including all RADICAL-Pilot components."""
         logger.info("RadicalPilotExecutor is terminating...")
+
+        self._terminate.set()
+
+        # ensure we are in the bulk submssion mode
+        if self.bulk_mode:
+            self._bulk_thread.join()
+
         self.session.close(download=True)
         logger.info("RadicalPilotExecutor is terminated.")
 

parsl/monitoring/monitoring.py

@@ -14,7 +14,7 @@ import typeguard
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringHubStartError
-from parsl.monitoring.radios import MultiprocessingQueueRadioSender
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.monitoring.router import router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue

parsl/monitoring/radios/base.py

@@ -0,0 +1,13 @@
+import logging
+from abc import ABCMeta, abstractmethod
+from typing import Optional
+
+_db_manager_excepts: Optional[Exception]
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRadioSender(metaclass=ABCMeta):
+    @abstractmethod
+    def send(self, message: object) -> None:
+        pass

parsl/monitoring/radios/filesystem.py

@@ -0,0 +1,52 @@
+import logging
+import os
+import pickle
+import uuid
+
+from parsl.monitoring.radios.base import MonitoringRadioSender
+
+logger = logging.getLogger(__name__)
+
+
+class FilesystemRadioSender(MonitoringRadioSender):
+    """A MonitoringRadioSender that sends messages over a shared filesystem.
+
+    The messsage directory structure is based on maildir,
+    https://en.wikipedia.org/wiki/Maildir
+
+    The writer creates a message in tmp/ and then when it is fully
+    written, moves it atomically into new/
+
+    The reader ignores tmp/ and only reads and deletes messages from
+    new/
+
+    This avoids a race condition of reading partially written messages.
+
+    This radio is likely to give higher shared filesystem load compared to
+    the UDP radio, but should be much more reliable.
+    """
+
+    def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
+        logger.info("filesystem based monitoring channel initializing")
+        self.base_path = f"{run_dir}/monitor-fs-radio/"
+        self.tmp_path = f"{self.base_path}/tmp"
+        self.new_path = f"{self.base_path}/new"
+
+        os.makedirs(self.tmp_path, exist_ok=True)
+        os.makedirs(self.new_path, exist_ok=True)
+
+    def send(self, message: object) -> None:
+        logger.info("Sending a monitoring message via filesystem")
+
+        unique_id = str(uuid.uuid4())
+
+        tmp_filename = f"{self.tmp_path}/{unique_id}"
+        new_filename = f"{self.new_path}/{unique_id}"
+        buffer = message
+
+        # this will write the message out then atomically
+        # move it into new/, so that a partially written
+        # file will never be observed in new/
+        with open(tmp_filename, "wb") as f:
+            pickle.dump(buffer, f)
+        os.rename(tmp_filename, new_filename)

parsl/monitoring/radios/htex.py

@@ -0,0 +1,57 @@
+import logging
+import pickle
+
+from parsl.monitoring.radios.base import MonitoringRadioSender
+
+logger = logging.getLogger(__name__)
+
+
+class HTEXRadioSender(MonitoringRadioSender):
+
+    def __init__(self, monitoring_url: str, timeout: int = 10):
+        """
+        Parameters
+        ----------
+
+        monitoring_url : str
+            URL of the form <scheme>://<IP>:<PORT>
+        timeout : int
+            timeout, default=10s
+        """
+        logger.info("htex-based monitoring channel initialising")
+
+    def send(self, message: object) -> None:
+        """ Sends a message to the UDP receiver
+
+        Parameter
+        ---------
+
+        message: object
+            Arbitrary pickle-able object that is to be sent
+
+        Returns:
+            None
+        """
+
+        import parsl.executors.high_throughput.monitoring_info
+
+        result_queue = parsl.executors.high_throughput.monitoring_info.result_queue
+
+        # this message needs to go in the result queue tagged so that it is treated
+        # i) as a monitoring message by the interchange, and then further more treated
+        # as a RESOURCE_INFO message when received by monitoring (rather than a NODE_INFO
+        # which is the implicit default for messages from the interchange)
+
+        # for the interchange, the outer wrapper, this needs to be a dict:
+
+        interchange_msg = {
+            'type': 'monitoring',
+            'payload': message
+        }
+
+        if result_queue:
+            result_queue.put(pickle.dumps(interchange_msg))
+        else:
+            logger.error("result_queue is uninitialized - cannot put monitoring message")
+
+        return

parsl/monitoring/radios/multiprocessing.py

@@ -0,0 +1,17 @@
+from multiprocessing.queues import Queue
+
+from parsl.monitoring.radios.base import MonitoringRadioSender
+
+
+class MultiprocessingQueueRadioSender(MonitoringRadioSender):
+    """A monitoring radio which connects over a multiprocessing Queue.
+    This radio is intended to be used on the submit side, where components
+    in the submit process, or processes launched by multiprocessing, will have
+    access to a Queue shared with the monitoring database code (bypassing the
+    monitoring router).
+    """
+    def __init__(self, queue: Queue) -> None:
+        self.queue = queue
+
+    def send(self, message: object) -> None:
+        self.queue.put(message)

parsl/monitoring/radios/udp.py

@@ -0,0 +1,56 @@
+import logging
+import pickle
+import socket
+
+from parsl.monitoring.radios.base import MonitoringRadioSender
+
+
+class UDPRadioSender(MonitoringRadioSender):
+
+    def __init__(self, monitoring_url: str, timeout: int = 10):
+        """
+        Parameters
+        ----------
+
+        monitoring_url : str
+            URL of the form <scheme>://<IP>:<PORT>
+        timeout : int
+            timeout, default=10s
+        """
+        self.monitoring_url = monitoring_url
+        self.sock_timeout = timeout
+        try:
+            self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
+            self.port = int(port)
+        except Exception:
+            raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
+
+        self.sock = socket.socket(socket.AF_INET,
+                                  socket.SOCK_DGRAM,
+                                  socket.IPPROTO_UDP)  # UDP
+        self.sock.settimeout(self.sock_timeout)
+
+    def send(self, message: object) -> None:
+        """ Sends a message to the UDP receiver
+
+        Parameter
+        ---------
+
+        message: object
+            Arbitrary pickle-able object that is to be sent
+
+        Returns:
+            None
+        """
+        try:
+            buffer = pickle.dumps(message)
+        except Exception:
+            logging.exception("Exception during pickling", exc_info=True)
+            return
+
+        try:
+            self.sock.sendto(buffer, (self.ip, self.port))
+        except socket.timeout:
+            logging.error("Could not send message within timeout limit")
+            return
+        return

parsl/monitoring/radios/zmq.py

@@ -0,0 +1,17 @@
+import zmq
+
+from parsl.monitoring.radios.base import MonitoringRadioSender
+
+
+class ZMQRadioSender(MonitoringRadioSender):
+    """A monitoring radio which connects over ZMQ. This radio is not
+    thread-safe, because its use of ZMQ is not thread-safe.
+    """
+
+    def __init__(self, hub_address: str, hub_zmq_port: int) -> None:
+        self._hub_channel = zmq.Context().socket(zmq.DEALER)
+        self._hub_channel.set_hwm(0)
+        self._hub_channel.connect(f"tcp://{hub_address}:{hub_zmq_port}")
+
+    def send(self, message: object) -> None:
+        self._hub_channel.send_pyobj(message)

parsl/monitoring/remote.py

@@ -7,12 +7,10 @@ from multiprocessing import Event
 from typing import Any, Callable, Dict, List, Sequence, Tuple
 
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.radios import (
-    FilesystemRadioSender,
-    HTEXRadioSender,
-    MonitoringRadioSender,
-    UDPRadioSender,
-)
+from parsl.monitoring.radios.base import MonitoringRadioSender
+from parsl.monitoring.radios.filesystem import FilesystemRadioSender
+from parsl.monitoring.radios.htex import HTEXRadioSender
+from parsl.monitoring.radios.udp import UDPRadioSender
 from parsl.multiprocessing import ForkProcess
 from parsl.process_loggers import wrap_with_logs
 

parsl/monitoring/router.py

@@ -14,7 +14,7 @@ import typeguard
 import zmq
 
 from parsl.log_utils import set_file_logger
-from parsl.monitoring.radios import MultiprocessingQueueRadioSender
+from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle

parsl/providers/cluster_provider.py

@@ -6,6 +6,7 @@ from parsl.launchers.base import Launcher
 from parsl.launchers.errors import BadLauncher
 from parsl.providers.base import ExecutionProvider
 from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError
+from parsl.utils import execute_wait
 
 logger = logging.getLogger(__name__)
 
@@ -17,8 +18,6 @@ class ClusterProvider(ExecutionProvider):
     ----------
       label : str
         Label for this provider.
-      channel : Channel
-        Channel for accessing this provider.
       walltime : str
         Walltime requested per block in HH:MM:SS.
       launcher : Launcher
@@ -44,7 +43,6 @@ class ClusterProvider(ExecutionProvider):
 
     def __init__(self,
                  label,
-                 channel,
                  nodes_per_block,
                  init_blocks,
                  min_blocks,
@@ -55,7 +53,6 @@ class ClusterProvider(ExecutionProvider):
                  cmd_timeout=10):
 
         self._label = label
-        self.channel = channel
         self.nodes_per_block = nodes_per_block
         self.init_blocks = init_blocks
         self.min_blocks = min_blocks
@@ -76,7 +73,7 @@ class ClusterProvider(ExecutionProvider):
         t = self.cmd_timeout
         if timeout is not None:
             t = timeout
-        return self.channel.execute_wait(cmd, t)
+        return execute_wait(cmd, t)
 
     def _write_submit_script(self, template, script_filename, job_name, configs):
         """Generate submit script and write it to a file.

parsl/providers/condor/condor.py

@@ -5,7 +5,6 @@ import time
 
 import typeguard
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
@@ -18,8 +17,6 @@ logger = logging.getLogger(__name__)
 
 from typing import Dict, List, Optional
 
-from parsl.channels.base import Channel
-
 # See http://pages.cs.wisc.edu/~adesmet/status.html
 translate_table = {
     '1': JobState.PENDING,
@@ -36,8 +33,6 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_slot : int
@@ -79,7 +74,6 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
     """
     @typeguard.typechecked
     def __init__(self,
-                 channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_slot: Optional[int] = None,
                  mem_per_slot: Optional[float] = None,
@@ -100,7 +94,6 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
 
         label = 'condor'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -226,7 +219,7 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
 
         job_config = {}
         job_config["job_name"] = job_name
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config["submit_script_dir"] = self.script_dir
         job_config["project"] = self.project
         job_config["nodes"] = self.nodes_per_block
         job_config["scheduler_options"] = scheduler_options

parsl/providers/grid_engine/grid_engine.py

@@ -2,7 +2,6 @@ import logging
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -36,8 +35,6 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     min_blocks : int
@@ -62,7 +59,6 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  init_blocks=1,
                  min_blocks=0,
@@ -76,7 +72,6 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
                  queue=None):
         label = 'grid_engine'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -100,7 +95,7 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
             self.nodes_per_block, tasks_per_node))
 
         job_config = {}
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config["submit_script_dir"] = self.script_dir
         job_config["nodes"] = self.nodes_per_block
         job_config["walltime"] = self.walltime
         job_config["scheduler_options"] = self.scheduler_options

parsl/providers/local/local.py

@@ -2,7 +2,6 @@ import logging
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.base import ExecutionProvider
@@ -11,7 +10,7 @@ from parsl.providers.errors import (
     ScriptPathError,
     SubmitException,
 )
-from parsl.utils import RepresentationMixin
+from parsl.utils import RepresentationMixin, execute_wait
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +36,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  launcher=SingleNodeLauncher(),
                  init_blocks=1,
@@ -46,7 +44,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
                  worker_init='',
                  cmd_timeout=30,
                  parallelism=1):
-        self.channel = channel
         self._label = 'local'
         self.nodes_per_block = nodes_per_block
         self.launcher = launcher
@@ -118,7 +115,7 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
         return [self.resources[jid]['status'] for jid in job_ids]
 
     def _is_alive(self, job_dict):
-        retcode, stdout, stderr = self.channel.execute_wait(
+        retcode, stdout, stderr = execute_wait(
             'ps -p {} > /dev/null 2> /dev/null; echo "STATUS:$?" '.format(
                 job_dict['remote_pid']), self.cmd_timeout)
         for line in stdout.split('\n'):
@@ -223,11 +220,11 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
         #      cancel the task later.
         #
         # We need to do the >/dev/null 2>&1 so that bash closes stdout, otherwise
-        # channel.execute_wait hangs reading the process stdout until all the
+        # execute_wait hangs reading the process stdout until all the
         # background commands complete.
         cmd = '/bin/bash -c \'echo - >{0}.ec && {{ {{ bash {0} 1>{0}.out 2>{0}.err ; ' \
               'echo $? > {0}.ec ; }} >/dev/null 2>&1 & echo "PID:$!" ; }}\''.format(script_path)
-        retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
+        retcode, stdout, stderr = execute_wait(cmd, self.cmd_timeout)
         if retcode != 0:
             raise SubmitException(job_name, "Launch command exited with code {0}".format(retcode),
                                   stdout, stderr)
@@ -258,7 +255,7 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
             job_dict['cancelled'] = True
             logger.debug("Terminating job/process ID: {0}".format(job))
             cmd = "kill -- -$(ps -o pgid= {} | grep -o '[0-9]*')".format(job_dict['remote_pid'])
-            retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
+            retcode, stdout, stderr = execute_wait(cmd, self.cmd_timeout)
             if retcode != 0:
                 logger.warning("Failed to kill PID: {} and child processes on {}".format(job_dict['remote_pid'],
                                                                                          self.label))

parsl/providers/lsf/lsf.py

@@ -3,7 +3,6 @@ import math
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.cluster_provider import ClusterProvider
@@ -32,8 +31,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
         When request_by_nodes is False, it is computed by cores_per_block / cores_per_node.
@@ -77,7 +74,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
     """
 
     def __init__(self,
-                 channel=LocalChannel(),
                  nodes_per_block=1,
                  cores_per_block=None,
                  cores_per_node=None,
@@ -96,7 +92,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
                  launcher=SingleNodeLauncher()):
         label = 'LSF'
         super().__init__(label,
-                         channel,
                          nodes_per_block,
                          init_blocks,
                          min_blocks,
@@ -211,7 +206,7 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
         logger.debug("Requesting one block with {} nodes".format(self.nodes_per_block))
 
         job_config = {}
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config["submit_script_dir"] = self.script_dir
         job_config["nodes"] = self.nodes_per_block
         job_config["tasks_per_node"] = tasks_per_node
         job_config["walltime"] = wtime_to_minutes(self.walltime)

parsl/providers/pbspro/pbspro.py

@@ -3,7 +3,6 @@ import logging
 import os
 import time
 
-from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.pbspro.template import template_string
@@ -17,8 +16,6 @@ class PBSProProvider(TorqueProvider):
 
     Parameters
     ----------
-    channel : Channel
-        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str
@@ -51,7 +48,6 @@ class PBSProProvider(TorqueProvider):
         :class:`~parsl.launchers.SingleNodeLauncher`.
     """
     def __init__(self,
-                 channel=LocalChannel(),
                  account=None,
                  queue=None,
                  scheduler_options='',
@@ -66,8 +62,7 @@ class PBSProProvider(TorqueProvider):
                  launcher=SingleNodeLauncher(),
                  walltime="00:20:00",
                  cmd_timeout=120):
-        super().__init__(channel,
-                         account,
+        super().__init__(account,
                          queue,
                          scheduler_options,
                          worker_init,
@@ -159,7 +154,7 @@ class PBSProProvider(TorqueProvider):
         )
 
         job_config = {}
-        job_config["submit_script_dir"] = self.channel.script_dir
+        job_config["submit_script_dir"] = self.script_dir
         job_config["nodes_per_block"] = self.nodes_per_block
         job_config["ncpus"] = self.cpus_per_node
         job_config["walltime"] = self.walltime