parsl 2024.11.25__py3-none-any.whl → 2024.12.9__py3-none-any.whl

parsl/monitoring/radios.py

@@ -1,191 +0,0 @@
-import logging
-import os
-import pickle
-import socket
-import uuid
-from abc import ABCMeta, abstractmethod
-from multiprocessing.queues import Queue
-
-import zmq
-
-logger = logging.getLogger(__name__)
-
-
-class MonitoringRadioSender(metaclass=ABCMeta):
-    @abstractmethod
-    def send(self, message: object) -> None:
-        pass
-
-
-class FilesystemRadioSender(MonitoringRadioSender):
-    """A MonitoringRadioSender that sends messages over a shared filesystem.
-
-    The messsage directory structure is based on maildir,
-    https://en.wikipedia.org/wiki/Maildir
-
-    The writer creates a message in tmp/ and then when it is fully
-    written, moves it atomically into new/
-
-    The reader ignores tmp/ and only reads and deletes messages from
-    new/
-
-    This avoids a race condition of reading partially written messages.
-
-    This radio is likely to give higher shared filesystem load compared to
-    the UDP radio, but should be much more reliable.
-    """
-
-    def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
-        logger.info("filesystem based monitoring channel initializing")
-        self.base_path = f"{run_dir}/monitor-fs-radio/"
-        self.tmp_path = f"{self.base_path}/tmp"
-        self.new_path = f"{self.base_path}/new"
-
-        os.makedirs(self.tmp_path, exist_ok=True)
-        os.makedirs(self.new_path, exist_ok=True)
-
-    def send(self, message: object) -> None:
-        logger.info("Sending a monitoring message via filesystem")
-
-        unique_id = str(uuid.uuid4())
-
-        tmp_filename = f"{self.tmp_path}/{unique_id}"
-        new_filename = f"{self.new_path}/{unique_id}"
-        buffer = message
-
-        # this will write the message out then atomically
-        # move it into new/, so that a partially written
-        # file will never be observed in new/
-        with open(tmp_filename, "wb") as f:
-            pickle.dump(buffer, f)
-        os.rename(tmp_filename, new_filename)
-
-
-class HTEXRadioSender(MonitoringRadioSender):
-
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
-
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        logger.info("htex-based monitoring channel initialising")
-
-    def send(self, message: object) -> None:
-        """ Sends a message to the UDP receiver
-
-        Parameter
-        ---------
-
-        message: object
-            Arbitrary pickle-able object that is to be sent
-
-        Returns:
-            None
-        """
-
-        import parsl.executors.high_throughput.monitoring_info
-
-        result_queue = parsl.executors.high_throughput.monitoring_info.result_queue
-
-        # this message needs to go in the result queue tagged so that it is treated
-        # i) as a monitoring message by the interchange, and then further more treated
-        # as a RESOURCE_INFO message when received by monitoring (rather than a NODE_INFO
-        # which is the implicit default for messages from the interchange)
-
-        # for the interchange, the outer wrapper, this needs to be a dict:
-
-        interchange_msg = {
-            'type': 'monitoring',
-            'payload': message
-        }
-
-        if result_queue:
-            result_queue.put(pickle.dumps(interchange_msg))
-        else:
-            logger.error("result_queue is uninitialized - cannot put monitoring message")
-
-        return
-
-
-class UDPRadioSender(MonitoringRadioSender):
-
-    def __init__(self, monitoring_url: str, timeout: int = 10):
-        """
-        Parameters
-        ----------
-
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        timeout : int
-            timeout, default=10s
-        """
-        self.monitoring_url = monitoring_url
-        self.sock_timeout = timeout
-        try:
-            self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
-            self.port = int(port)
-        except Exception:
-            raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
-
-        self.sock = socket.socket(socket.AF_INET,
-                                  socket.SOCK_DGRAM,
-                                  socket.IPPROTO_UDP)  # UDP
-        self.sock.settimeout(self.sock_timeout)
-
-    def send(self, message: object) -> None:
-        """ Sends a message to the UDP receiver
-
-        Parameter
-        ---------
-
-        message: object
-            Arbitrary pickle-able object that is to be sent
-
-        Returns:
-            None
-        """
-        try:
-            buffer = pickle.dumps(message)
-        except Exception:
-            logging.exception("Exception during pickling", exc_info=True)
-            return
-
-        try:
-            self.sock.sendto(buffer, (self.ip, self.port))
-        except socket.timeout:
-            logging.error("Could not send message within timeout limit")
-            return
-        return
-
-
-class MultiprocessingQueueRadioSender(MonitoringRadioSender):
-    """A monitoring radio which connects over a multiprocessing Queue.
-    This radio is intended to be used on the submit side, where components
-    in the submit process, or processes launched by multiprocessing, will have
-    access to a Queue shared with the monitoring database code (bypassing the
-    monitoring router).
-    """
-    def __init__(self, queue: Queue) -> None:
-        self.queue = queue
-
-    def send(self, message: object) -> None:
-        self.queue.put(message)
-
-
-class ZMQRadioSender(MonitoringRadioSender):
-    """A monitoring radio which connects over ZMQ. This radio is not
-    thread-safe, because its use of ZMQ is not thread-safe.
-    """
-
-    def __init__(self, hub_address: str, hub_zmq_port: int) -> None:
-        self._hub_channel = zmq.Context().socket(zmq.DEALER)
-        self._hub_channel.set_hwm(0)
-        self._hub_channel.connect(f"tcp://{hub_address}:{hub_zmq_port}")
-
-    def send(self, message: object) -> None:
-        self._hub_channel.send_pyobj(message)

parsl/tests/test_channels/test_large_output.py

@@ -1,22 +0,0 @@
-import pytest
-
-from parsl.channels.local.local import LocalChannel
-
-
-@pytest.mark.local
-def test_local_large_output_2210():
-    """Regression test for #2210.
-    The local channel was hanging if the specified command gave too
-    much output, due to a race condition between process exiting and
-    pipes filling up.
-    """
-
-    c = LocalChannel()
-
-    # this will output 128kb of stdout
-    c.execute_wait("yes | dd count=128 bs=1024", walltime=60)
-
-    # if this test fails, execute_wait should raise a timeout
-    # exception.
-
-    # The contents out the output is not verified by this test

parsl/tests/test_channels/test_local_channel.py

@@ -1,19 +0,0 @@
-import pytest
-
-from parsl.channels.local.local import LocalChannel
-
-
-@pytest.mark.local
-def test_env():
-    ''' Regression testing for issue #27
-    '''
-
-    lc = LocalChannel()
-    rc, stdout, stderr = lc.execute_wait("env", 1)
-
-    stdout = stdout.split('\n')
-    x = [s for s in stdout if s.startswith("PATH=")]
-    assert x, "PATH not found"
-
-    x = [s for s in stdout if s.startswith("HOME=")]
-    assert x, "HOME not found"