parsl 2024.6.10__py3-none-any.whl → 2024.6.17__py3-none-any.whl

parsl/app/app.py

@@ -66,8 +66,6 @@ class AppBase(metaclass=ABCMeta):
             self.kwargs['walltime'] = params['walltime'].default
         if 'parsl_resource_specification' in params:
             self.kwargs['parsl_resource_specification'] = params['parsl_resource_specification'].default
-        self.outputs = params['outputs'].default if 'outputs' in params else []
-        self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
     def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:

parsl/executors/high_throughput/executor.py

@@ -1,13 +1,13 @@
 import logging
 import math
 import pickle
+import subprocess
 import threading
 import typing
 import warnings
 from collections import defaultdict
 from concurrent.futures import Future
 from dataclasses import dataclass
-from multiprocessing import Process
 from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import typeguard
@@ -18,7 +18,7 @@ from parsl.addresses import get_all_addresses
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.staging import Staging
 from parsl.executors.errors import BadMessage, ScalingFailed
-from parsl.executors.high_throughput import interchange, zmq_pipes
+from parsl.executors.high_throughput import zmq_pipes
 from parsl.executors.high_throughput.errors import CommandClientTimeoutError
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
@@ -26,7 +26,6 @@ from parsl.executors.high_throughput.mpi_prefix_composer import (
 )
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
-from parsl.multiprocessing import ForkProcess
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import LocalProvider
 from parsl.providers.base import ExecutionProvider
@@ -305,7 +304,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self._task_counter = 0
         self.worker_ports = worker_ports
         self.worker_port_range = worker_port_range
-        self.interchange_proc: Optional[Process] = None
+        self.interchange_proc: Optional[subprocess.Popen] = None
         self.interchange_port_range = interchange_port_range
         self.heartbeat_threshold = heartbeat_threshold
         self.heartbeat_period = heartbeat_period
@@ -520,38 +519,45 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
         logger.info("Queue management worker finished")
 
-    def _start_local_interchange_process(self):
+    def _start_local_interchange_process(self) -> None:
         """ Starts the interchange process locally
 
-        Starts the interchange process locally and uses an internal command queue to
+        Starts the interchange process locally and uses the command queue to
         get the worker task and result ports that the interchange has bound to.
         """
-        self.interchange_proc = ForkProcess(target=interchange.starter,
-                                            kwargs={"client_address": "127.0.0.1",
-                                                    "client_ports": (self.outgoing_q.port,
-                                                                     self.incoming_q.port,
-                                                                     self.command_client.port),
-                                                    "interchange_address": self.address,
-                                                    "worker_ports": self.worker_ports,
-                                                    "worker_port_range": self.worker_port_range,
-                                                    "hub_address": self.hub_address,
-                                                    "hub_zmq_port": self.hub_zmq_port,
-                                                    "logdir": self.logdir,
-                                                    "heartbeat_threshold": self.heartbeat_threshold,
-                                                    "poll_period": self.poll_period,
-                                                    "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
-                                                    "cert_dir": self.cert_dir,
-                                                    },
-                                            daemon=True,
-                                            name="HTEX-Interchange"
-                                            )
-        self.interchange_proc.start()
 
+        interchange_config = {"client_address": "127.0.0.1",
+                              "client_ports": (self.outgoing_q.port,
+                                               self.incoming_q.port,
+                                               self.command_client.port),
+                              "interchange_address": self.address,
+                              "worker_ports": self.worker_ports,
+                              "worker_port_range": self.worker_port_range,
+                              "hub_address": self.hub_address,
+                              "hub_zmq_port": self.hub_zmq_port,
+                              "logdir": self.logdir,
+                              "heartbeat_threshold": self.heartbeat_threshold,
+                              "poll_period": self.poll_period,
+                              "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
+                              "cert_dir": self.cert_dir,
+                              }
+
+        config_pickle = pickle.dumps(interchange_config)
+
+        self.interchange_proc = subprocess.Popen(b"interchange.py", stdin=subprocess.PIPE)
+        stdin = self.interchange_proc.stdin
+        assert stdin is not None, "Popen should have created an IO object (vs default None) because of PIPE mode"
+
+        logger.debug("Popened interchange process. Writing config object")
+        stdin.write(config_pickle)
+        stdin.flush()
+        logger.debug("Sent config object. Requesting worker ports")
         try:
             (self.worker_task_port, self.worker_result_port) = self.command_client.run("WORKER_PORTS", timeout_s=120)
         except CommandClientTimeoutError:
-            logger.error("Interchange has not completed initialization in 120s. Aborting")
+            logger.error("Interchange has not completed initialization. Aborting")
             raise Exception("Interchange failed to start")
+        logger.debug("Got worker ports")
 
     def _start_queue_management_thread(self):
         """Method to start the management thread as a daemon.
@@ -810,13 +816,12 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         logger.info("Attempting HighThroughputExecutor shutdown")
 
         self.interchange_proc.terminate()
-        self.interchange_proc.join(timeout=timeout)
-        if self.interchange_proc.is_alive():
+        try:
+            self.interchange_proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
             logger.info("Unable to terminate Interchange process; sending SIGKILL")
             self.interchange_proc.kill()
 
-        self.interchange_proc.close()
-
         logger.info("Finished HighThroughputExecutor shutdown attempt")
 
     def get_usage_information(self):

parsl/executors/high_throughput/interchange.py

@@ -672,13 +672,10 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
     logger.addHandler(handler)
 
 
-@wrap_with_logs(target="interchange")
-def starter(*args: Any, **kwargs: Any) -> None:
-    """Start the interchange process
-
-    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
-    """
+if __name__ == "__main__":
     setproctitle("parsl: HTEX interchange")
-    # logger = multiprocessing.get_logger()
-    ic = Interchange(*args, **kwargs)
+
+    config = pickle.load(sys.stdin.buffer)
+
+    ic = Interchange(**config)
     ic.start()

parsl/providers/kubernetes/kube.py

@@ -243,13 +243,13 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         for jid in to_poll_job_ids:
             phase = None
             try:
-                pod_status = self.kube_client.read_namespaced_pod_status(name=jid, namespace=self.namespace)
+                pod = self.kube_client.read_namespaced_pod(name=jid, namespace=self.namespace)
             except Exception:
                 logger.exception("Failed to poll pod {} status, most likely because pod was terminated".format(jid))
                 if self.resources[jid]['status'] is JobStatus(JobState.RUNNING):
                     phase = 'Unknown'
             else:
-                phase = pod_status.status.phase
+                phase = pod.status.phase
             if phase:
                 status = translate_table.get(phase, JobState.UNKNOWN)
                 logger.debug("Updating pod {} with status {} to parsl status {}".format(jid,
@@ -286,7 +286,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         # Create the environment variables and command to initiate IPP
         environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
 
-        launch_args = ["-c", "{0};".format(cmd_string)]
+        launch_args = ["-c", "{0}".format(cmd_string)]
 
         volume_mounts = []
         # Create mount paths for the volumes

parsl/tests/test_htex/test_htex.py

@@ -1,11 +1,11 @@
 import pathlib
 import warnings
+from subprocess import Popen, TimeoutExpired
 from unittest import mock
 
 import pytest
 
 from parsl import HighThroughputExecutor, curvezmq
-from parsl.multiprocessing import ForkProcess
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -78,16 +78,33 @@ def test_htex_shutdown(
     timeout_expires: bool,
     htex: HighThroughputExecutor,
 ):
-    mock_ix_proc = mock.Mock(spec=ForkProcess)
+    mock_ix_proc = mock.Mock(spec=Popen)
 
     if started:
         htex.interchange_proc = mock_ix_proc
-        mock_ix_proc.is_alive.return_value = True
+
+    # This will, in the absence of any exit trigger, block forever if
+    # no timeout is given and if the interchange does not terminate.
+    # Raise an exception to report that, rather than actually block,
+    # and hope that nothing is catching that exception.
+
+    # this function implements the behaviour if the interchange has
+    # not received a termination call
+    def proc_wait_alive(timeout):
+        if timeout:
+            raise TimeoutExpired(cmd="mock-interchange", timeout=timeout)
+        else:
+            raise RuntimeError("This wait call would hang forever")
+
+    def proc_wait_terminated(timeout):
+        return 0
+
+    mock_ix_proc.wait.side_effect = proc_wait_alive
 
     if not timeout_expires:
         # Simulate termination of the Interchange process
         def kill_interchange(*args, **kwargs):
-            mock_ix_proc.is_alive.return_value = False
+            mock_ix_proc.wait.side_effect = proc_wait_terminated
 
         mock_ix_proc.terminate.side_effect = kill_interchange
 
@@ -96,8 +113,8 @@ def test_htex_shutdown(
     mock_logs = mock_logger.info.call_args_list
     if started:
         assert mock_ix_proc.terminate.called
-        assert mock_ix_proc.join.called
-        assert {"timeout": 10} == mock_ix_proc.join.call_args[1]
+        assert mock_ix_proc.wait.called
+        assert {"timeout": 10} == mock_ix_proc.wait.call_args[1]
         if timeout_expires:
             assert "Unable to terminate Interchange" in mock_logs[1][0][0]
             assert mock_ix_proc.kill.called
@@ -105,7 +122,7 @@ def test_htex_shutdown(
         assert "Finished" in mock_logs[-1][0][0]
     else:
         assert not mock_ix_proc.terminate.called
-        assert not mock_ix_proc.join.called
+        assert not mock_ix_proc.wait.called
         assert "has not started" in mock_logs[0][0][0]
 
 

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.06.10'
+VERSION = '2024.06.17'

parsl-2024.6.17.data/scripts/interchange.py

@@ -0,0 +1,681 @@
+#!python
+import datetime
+import json
+import logging
+import os
+import pickle
+import platform
+import queue
+import random
+import signal
+import sys
+import threading
+import time
+from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
+
+import zmq
+
+from parsl import curvezmq
+from parsl.app.errors import RemoteExceptionWrapper
+from parsl.executors.high_throughput.manager_record import ManagerRecord
+from parsl.monitoring.message_type import MessageType
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize as serialize_object
+from parsl.utils import setproctitle
+from parsl.version import VERSION as PARSL_VERSION
+
+PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1)
+PKL_DRAINED_CODE = pickle.dumps((2 ** 32) - 2)
+
+LOGGER_NAME = "interchange"
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class ManagerLost(Exception):
+    ''' Task lost due to manager loss. Manager is considered lost when multiple heartbeats
+    have been missed.
+    '''
+    def __init__(self, manager_id: bytes, hostname: str) -> None:
+        self.manager_id = manager_id
+        self.tstamp = time.time()
+        self.hostname = hostname
+
+    def __str__(self) -> str:
+        return "Task failure due to loss of manager {} on host {}".format(self.manager_id.decode(), self.hostname)
+
+
+class VersionMismatch(Exception):
+    ''' Manager and Interchange versions do not match
+    '''
+    def __init__(self, interchange_version: str, manager_version: str):
+        self.interchange_version = interchange_version
+        self.manager_version = manager_version
+
+    def __str__(self) -> str:
+        return "Manager version info {} does not match interchange version info {}, causing a critical failure".format(
+            self.manager_version,
+            self.interchange_version)
+
+
+class Interchange:
+    """ Interchange is a task orchestrator for distributed systems.
+
+    1. Asynchronously queue large volume of tasks (>100K)
+    2. Allow for workers to join and leave the union
+    3. Detect workers that have failed using heartbeats
+    """
+    def __init__(self,
+                 *,
+                 client_address: str,
+                 interchange_address: Optional[str],
+                 client_ports: Tuple[int, int, int],
+                 worker_ports: Optional[Tuple[int, int]],
+                 worker_port_range: Tuple[int, int],
+                 hub_address: Optional[str],
+                 hub_zmq_port: Optional[int],
+                 heartbeat_threshold: int,
+                 logdir: str,
+                 logging_level: int,
+                 poll_period: int,
+                 cert_dir: Optional[str],
+                 ) -> None:
+        """
+        Parameters
+        ----------
+        client_address : str
+             The ip address at which the parsl client can be reached. Default: "127.0.0.1"
+
+        interchange_address : Optional str
+             If specified the interchange will only listen on this address for connections from workers
+             else, it binds to all addresses.
+
+        client_ports : triple(int, int, int)
+             The ports at which the client can be reached
+
+        worker_ports : tuple(int, int)
+             The specific two ports at which workers will connect to the Interchange.
+
+        worker_port_range : tuple(int, int)
+             The interchange picks ports at random from the range which will be used by workers.
+             This is overridden when the worker_ports option is set.
+
+        hub_address : str
+             The IP address at which the interchange can send info about managers to when monitoring is enabled.
+             When None, monitoring is disabled.
+
+        hub_zmq_port : str
+             The port at which the interchange can send info about managers to when monitoring is enabled.
+             When None, monitoring is disabled.
+
+        heartbeat_threshold : int
+             Number of seconds since the last heartbeat after which worker is considered lost.
+
+        logdir : str
+             Parsl log directory paths. Logs and temp files go here.
+
+        logging_level : int
+             Logging level as defined in the logging module.
+
+        poll_period : int
+             The main thread polling period, in milliseconds.
+
+        cert_dir : str | None
+            Path to the certificate directory.
+        """
+        self.cert_dir = cert_dir
+        self.logdir = logdir
+        os.makedirs(self.logdir, exist_ok=True)
+
+        start_file_logger("{}/interchange.log".format(self.logdir), level=logging_level)
+        logger.propagate = False
+        logger.debug("Initializing Interchange process")
+
+        self.client_address = client_address
+        self.interchange_address: str = interchange_address or "*"
+        self.poll_period = poll_period
+
+        logger.info("Attempting connection to client at {} on ports: {},{},{}".format(
+            client_address, client_ports[0], client_ports[1], client_ports[2]))
+        self.zmq_context = curvezmq.ServerContext(self.cert_dir)
+        self.task_incoming = self.zmq_context.socket(zmq.DEALER)
+        self.task_incoming.set_hwm(0)
+        self.task_incoming.connect("tcp://{}:{}".format(client_address, client_ports[0]))
+        self.results_outgoing = self.zmq_context.socket(zmq.DEALER)
+        self.results_outgoing.set_hwm(0)
+        self.results_outgoing.connect("tcp://{}:{}".format(client_address, client_ports[1]))
+
+        self.command_channel = self.zmq_context.socket(zmq.REP)
+        self.command_channel.connect("tcp://{}:{}".format(client_address, client_ports[2]))
+        logger.info("Connected to client")
+
+        self.hub_address = hub_address
+        self.hub_zmq_port = hub_zmq_port
+
+        self.pending_task_queue: queue.Queue[Any] = queue.Queue(maxsize=10 ** 6)
+        self.count = 0
+
+        self.worker_ports = worker_ports
+        self.worker_port_range = worker_port_range
+
+        self.task_outgoing = self.zmq_context.socket(zmq.ROUTER)
+        self.task_outgoing.set_hwm(0)
+        self.results_incoming = self.zmq_context.socket(zmq.ROUTER)
+        self.results_incoming.set_hwm(0)
+
+        if self.worker_ports:
+            self.worker_task_port = self.worker_ports[0]
+            self.worker_result_port = self.worker_ports[1]
+
+            self.task_outgoing.bind(f"tcp://{self.interchange_address}:{self.worker_task_port}")
+            self.results_incoming.bind(f"tcp://{self.interchange_address}:{self.worker_result_port}")
+
+        else:
+            self.worker_task_port = self.task_outgoing.bind_to_random_port(f"tcp://{self.interchange_address}",
+                                                                           min_port=worker_port_range[0],
+                                                                           max_port=worker_port_range[1], max_tries=100)
+            self.worker_result_port = self.results_incoming.bind_to_random_port(f"tcp://{self.interchange_address}",
+                                                                                min_port=worker_port_range[0],
+                                                                                max_port=worker_port_range[1], max_tries=100)
+
+        logger.info("Bound to ports {},{} for incoming worker connections".format(
+            self.worker_task_port, self.worker_result_port))
+
+        self._ready_managers: Dict[bytes, ManagerRecord] = {}
+        self.connected_block_history: List[str] = []
+
+        self.heartbeat_threshold = heartbeat_threshold
+
+        self.current_platform = {'parsl_v': PARSL_VERSION,
+                                 'python_v': "{}.{}.{}".format(sys.version_info.major,
+                                                               sys.version_info.minor,
+                                                               sys.version_info.micro),
+                                 'os': platform.system(),
+                                 'hostname': platform.node(),
+                                 'dir': os.getcwd()}
+
+        logger.info("Platform info: {}".format(self.current_platform))
+
+    def get_tasks(self, count: int) -> Sequence[dict]:
+        """ Obtains a batch of tasks from the internal pending_task_queue
+
+        Parameters
+        ----------
+        count: int
+            Count of tasks to get from the queue
+
+        Returns
+        -------
+        List of upto count tasks. May return fewer than count down to an empty list
+            eg. [{'task_id':<x>, 'buffer':<buf>} ... ]
+        """
+        tasks = []
+        for _ in range(0, count):
+            try:
+                x = self.pending_task_queue.get(block=False)
+            except queue.Empty:
+                break
+            else:
+                tasks.append(x)
+
+        return tasks
+
+    @wrap_with_logs(target="interchange")
+    def task_puller(self) -> NoReturn:
+        """Pull tasks from the incoming tasks zmq pipe onto the internal
+        pending task queue
+        """
+        logger.info("Starting")
+        task_counter = 0
+
+        while True:
+            logger.debug("launching recv_pyobj")
+            try:
+                msg = self.task_incoming.recv_pyobj()
+            except zmq.Again:
+                # We just timed out while attempting to receive
+                logger.debug("zmq.Again with {} tasks in internal queue".format(self.pending_task_queue.qsize()))
+                continue
+
+            logger.debug("putting message onto pending_task_queue")
+            self.pending_task_queue.put(msg)
+            task_counter += 1
+            logger.debug(f"Fetched {task_counter} tasks so far")
+
+    def _create_monitoring_channel(self) -> Optional[zmq.Socket]:
+        if self.hub_address and self.hub_zmq_port:
+            logger.info("Connecting to MonitoringHub")
+            # This is a one-off because monitoring is unencrypted
+            hub_channel = zmq.Context().socket(zmq.DEALER)
+            hub_channel.set_hwm(0)
+            hub_channel.connect("tcp://{}:{}".format(self.hub_address, self.hub_zmq_port))
+            logger.info("Connected to MonitoringHub")
+            return hub_channel
+        else:
+            return None
+
+    def _send_monitoring_info(self, hub_channel: Optional[zmq.Socket], manager: ManagerRecord) -> None:
+        if hub_channel:
+            logger.info("Sending message {} to MonitoringHub".format(manager))
+
+            d: Dict = cast(Dict, manager.copy())
+            d['timestamp'] = datetime.datetime.now()
+            d['last_heartbeat'] = datetime.datetime.fromtimestamp(d['last_heartbeat'])
+
+            hub_channel.send_pyobj((MessageType.NODE_INFO, d))
+
+    @wrap_with_logs(target="interchange")
+    def _command_server(self) -> NoReturn:
+        """ Command server to run async command to the interchange
+        """
+        logger.debug("Command Server Starting")
+
+        # Need to create a new ZMQ socket for command server thread
+        hub_channel = self._create_monitoring_channel()
+
+        reply: Any  # the type of reply depends on the command_req received (aka this needs dependent types...)
+
+        while True:
+            try:
+                command_req = self.command_channel.recv_pyobj()
+                logger.debug("Received command request: {}".format(command_req))
+                if command_req == "OUTSTANDING_C":
+                    outstanding = self.pending_task_queue.qsize()
+                    for manager in self._ready_managers.values():
+                        outstanding += len(manager['tasks'])
+                    reply = outstanding
+
+                elif command_req == "CONNECTED_BLOCKS":
+                    reply = self.connected_block_history
+
+                elif command_req == "WORKERS":
+                    num_workers = 0
+                    for manager in self._ready_managers.values():
+                        num_workers += manager['worker_count']
+                    reply = num_workers
+
+                elif command_req == "MANAGERS":
+                    reply = []
+                    for manager_id in self._ready_managers:
+                        m = self._ready_managers[manager_id]
+                        idle_since = m['idle_since']
+                        if idle_since is not None:
+                            idle_duration = time.time() - idle_since
+                        else:
+                            idle_duration = 0.0
+                        resp = {'manager': manager_id.decode('utf-8'),
+                                'block_id': m['block_id'],
+                                'worker_count': m['worker_count'],
+                                'tasks': len(m['tasks']),
+                                'idle_duration': idle_duration,
+                                'active': m['active'],
+                                'parsl_version': m['parsl_version'],
+                                'python_version': m['python_version'],
+                                'draining': m['draining']}
+                        reply.append(resp)
+
+                elif command_req.startswith("HOLD_WORKER"):
+                    cmd, s_manager = command_req.split(';')
+                    manager_id = s_manager.encode('utf-8')
+                    logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
+                    if manager_id in self._ready_managers:
+                        m = self._ready_managers[manager_id]
+                        m['active'] = False
+                        self._send_monitoring_info(hub_channel, m)
+                    else:
+                        logger.warning("Worker to hold was not in ready managers list")
+
+                    reply = None
+
+                elif command_req == "WORKER_PORTS":
+                    reply = (self.worker_task_port, self.worker_result_port)
+
+                else:
+                    logger.error(f"Received unknown command: {command_req}")
+                    reply = None
+
+                logger.debug("Reply: {}".format(reply))
+                self.command_channel.send_pyobj(reply)
+
+            except zmq.Again:
+                logger.debug("Command thread is alive")
+                continue
+
+    @wrap_with_logs
+    def start(self) -> None:
+        """ Start the interchange
+        """
+
+        # If a user workflow has set its own signal handler for sigterm, that
+        # handler will be inherited by the interchange process because it is
+        # launched as a multiprocessing fork process.
+        # That can interfere with the interchange shutdown mechanism, which is
+        # to receive a SIGTERM and exit immediately.
+        # See Parsl issue #2343 (Threads and multiprocessing cannot be
+        # intermingled without deadlocks) which talks about other fork-related
+        # parent-process-inheritance problems.
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+        logger.info("Incoming ports bound")
+
+        hub_channel = self._create_monitoring_channel()
+
+        poll_period = self.poll_period
+
+        start = time.time()
+
+        self._task_puller_thread = threading.Thread(target=self.task_puller,
+                                                    name="Interchange-Task-Puller",
+                                                    daemon=True)
+        self._task_puller_thread.start()
+
+        self._command_thread = threading.Thread(target=self._command_server,
+                                                name="Interchange-Command",
+                                                daemon=True)
+        self._command_thread.start()
+
+        kill_event = threading.Event()
+
+        poller = zmq.Poller()
+        poller.register(self.task_outgoing, zmq.POLLIN)
+        poller.register(self.results_incoming, zmq.POLLIN)
+
+        # These are managers which we should examine in an iteration
+        # for scheduling a job (or maybe any other attention?).
+        # Anything altering the state of the manager should add it
+        # onto this list.
+        interesting_managers: Set[bytes] = set()
+
+        while not kill_event.is_set():
+            self.socks = dict(poller.poll(timeout=poll_period))
+
+            self.process_task_outgoing_incoming(interesting_managers, hub_channel, kill_event)
+            self.process_results_incoming(interesting_managers, hub_channel)
+            self.expire_bad_managers(interesting_managers, hub_channel)
+            self.expire_drained_managers(interesting_managers, hub_channel)
+            self.process_tasks_to_send(interesting_managers)
+
+        self.zmq_context.destroy()
+        delta = time.time() - start
+        logger.info("Processed {} tasks in {} seconds".format(self.count, delta))
+        logger.warning("Exiting")
+
+    def process_task_outgoing_incoming(
+            self,
+            interesting_managers: Set[bytes],
+            hub_channel: Optional[zmq.Socket],
+            kill_event: threading.Event
+    ) -> None:
+        """Process one message from manager on the task_outgoing channel.
+        Note that this message flow is in contradiction to the name of the
+        channel - it is not an outgoing message and it is not a task.
+        """
+        if self.task_outgoing in self.socks and self.socks[self.task_outgoing] == zmq.POLLIN:
+            logger.debug("starting task_outgoing section")
+            message = self.task_outgoing.recv_multipart()
+            manager_id = message[0]
+
+            try:
+                msg = json.loads(message[1].decode('utf-8'))
+            except Exception:
+                logger.warning("Got Exception reading message from manager: {!r}".format(
+                    manager_id), exc_info=True)
+                logger.debug("Message: \n{!r}\n".format(message[1]))
+                return
+
+            # perform a bit of validation on the structure of the deserialized
+            # object, at least enough to behave like a deserialization error
+            # in obviously malformed cases
+            if not isinstance(msg, dict) or 'type' not in msg:
+                logger.error(f"JSON message was not correctly formatted from manager: {manager_id!r}")
+                logger.debug("Message: \n{!r}\n".format(message[1]))
+                return
+
+            if msg['type'] == 'registration':
+                # We set up an entry only if registration works correctly
+                self._ready_managers[manager_id] = {'last_heartbeat': time.time(),
+                                                    'idle_since': time.time(),
+                                                    'block_id': None,
+                                                    'max_capacity': 0,
+                                                    'worker_count': 0,
+                                                    'active': True,
+                                                    'draining': False,
+                                                    'parsl_version': msg['parsl_v'],
+                                                    'python_version': msg['python_v'],
+                                                    'tasks': []}
+                self.connected_block_history.append(msg['block_id'])
+
+                interesting_managers.add(manager_id)
+                logger.info("Adding manager: {!r} to ready queue".format(manager_id))
+                m = self._ready_managers[manager_id]
+
+                # m is a ManagerRecord, but msg is a dict[Any,Any] and so can
+                # contain arbitrary fields beyond those in ManagerRecord (and
+                # indeed does - for example, python_v) which are then ignored
+                # later.
+                m.update(msg)  # type: ignore[typeddict-item]
+
+                logger.info("Registration info for manager {!r}: {}".format(manager_id, msg))
+                self._send_monitoring_info(hub_channel, m)
+
+                if (msg['python_v'].rsplit(".", 1)[0] != self.current_platform['python_v'].rsplit(".", 1)[0] or
+                    msg['parsl_v'] != self.current_platform['parsl_v']):
+                    logger.error("Manager {!r} has incompatible version info with the interchange".format(manager_id))
+                    logger.debug("Setting kill event")
+                    kill_event.set()
+                    e = VersionMismatch("py.v={} parsl.v={}".format(self.current_platform['python_v'].rsplit(".", 1)[0],
+                                                                    self.current_platform['parsl_v']),
+                                        "py.v={} parsl.v={}".format(msg['python_v'].rsplit(".", 1)[0],
+                                                                    msg['parsl_v'])
+                                        )
+                    result_package = {'type': 'result', 'task_id': -1, 'exception': serialize_object(e)}
+                    pkl_package = pickle.dumps(result_package)
+                    self.results_outgoing.send(pkl_package)
+                    logger.error("Sent failure reports, shutting down interchange")
+                else:
+                    logger.info("Manager {!r} has compatible Parsl version {}".format(manager_id, msg['parsl_v']))
+                    logger.info("Manager {!r} has compatible Python version {}".format(manager_id,
+                                                                                       msg['python_v'].rsplit(".", 1)[0]))
+            elif msg['type'] == 'heartbeat':
+                self._ready_managers[manager_id]['last_heartbeat'] = time.time()
+                logger.debug("Manager {!r} sent heartbeat via tasks connection".format(manager_id))
+                self.task_outgoing.send_multipart([manager_id, b'', PKL_HEARTBEAT_CODE])
+            elif msg['type'] == 'drain':
+                self._ready_managers[manager_id]['draining'] = True
+                logger.debug(f"Manager {manager_id!r} requested drain")
+            else:
+                logger.error(f"Unexpected message type received from manager: {msg['type']}")
+            logger.debug("leaving task_outgoing section")
+
+    def expire_drained_managers(self, interesting_managers: Set[bytes], hub_channel: Optional[zmq.Socket]) -> None:
+
+        for manager_id in list(interesting_managers):
+            # is it always true that a draining manager will be in interesting managers?
+            # i think so because it will have outstanding capacity?
+            m = self._ready_managers[manager_id]
+            if m['draining'] and len(m['tasks']) == 0:
+                logger.info(f"Manager {manager_id!r} is drained - sending drained message to manager")
+                self.task_outgoing.send_multipart([manager_id, b'', PKL_DRAINED_CODE])
+                interesting_managers.remove(manager_id)
+                self._ready_managers.pop(manager_id)
+
+                m['active'] = False
+                self._send_monitoring_info(hub_channel, m)
+
+    def process_tasks_to_send(self, interesting_managers: Set[bytes]) -> None:
+        # Check if there are tasks that could be sent to managers
+
+        logger.debug("Managers count (interesting/total): {interesting}/{total}".format(
+            total=len(self._ready_managers),
+            interesting=len(interesting_managers)))
+
+        if interesting_managers and not self.pending_task_queue.empty():
+            shuffled_managers = list(interesting_managers)
+            random.shuffle(shuffled_managers)
+
+            while shuffled_managers and not self.pending_task_queue.empty():  # cf. the if statement above...
+                manager_id = shuffled_managers.pop()
+                m = self._ready_managers[manager_id]
+                tasks_inflight = len(m['tasks'])
+                real_capacity = m['max_capacity'] - tasks_inflight
+
+                if (real_capacity and m['active'] and not m['draining']):
+                    tasks = self.get_tasks(real_capacity)
+                    if tasks:
+                        self.task_outgoing.send_multipart([manager_id, b'', pickle.dumps(tasks)])
+                        task_count = len(tasks)
+                        self.count += task_count
+                        tids = [t['task_id'] for t in tasks]
+                        m['tasks'].extend(tids)
+                        m['idle_since'] = None
+                        logger.debug("Sent tasks: {} to manager {!r}".format(tids, manager_id))
+                        # recompute real_capacity after sending tasks
+                        real_capacity = m['max_capacity'] - tasks_inflight
+                        if real_capacity > 0:
+                            logger.debug("Manager {!r} has free capacity {}".format(manager_id, real_capacity))
+                            # ... so keep it in the interesting_managers list
+                        else:
+                            logger.debug("Manager {!r} is now saturated".format(manager_id))
+                            interesting_managers.remove(manager_id)
+                else:
+                    interesting_managers.remove(manager_id)
+                    # logger.debug("Nothing to send to manager {}".format(manager_id))
+            logger.debug("leaving _ready_managers section, with {} managers still interesting".format(len(interesting_managers)))
+        else:
+            logger.debug("either no interesting managers or no tasks, so skipping manager pass")
+
+    def process_results_incoming(self, interesting_managers: Set[bytes], hub_channel: Optional[zmq.Socket]) -> None:
+        # Receive any results and forward to client
+        if self.results_incoming in self.socks and self.socks[self.results_incoming] == zmq.POLLIN:
+            logger.debug("entering results_incoming section")
+            manager_id, *all_messages = self.results_incoming.recv_multipart()
+            if manager_id not in self._ready_managers:
+                logger.warning("Received a result from a un-registered manager: {!r}".format(manager_id))
+            else:
+                logger.debug(f"Got {len(all_messages)} result items in batch from manager {manager_id!r}")
+
+                b_messages = []
+
+                for p_message in all_messages:
+                    r = pickle.loads(p_message)
+                    if r['type'] == 'result':
+                        # process this for task ID and forward to executor
+                        b_messages.append((p_message, r))
+                    elif r['type'] == 'monitoring':
+                        # the monitoring code makes the assumption that no
+                        # monitoring messages will be received if monitoring
+                        # is not configured, and that hub_channel will only
+                        # be None when monitoring is not configurated.
+                        assert hub_channel is not None
+
+                        hub_channel.send_pyobj(r['payload'])
+                    elif r['type'] == 'heartbeat':
+                        logger.debug(f"Manager {manager_id!r} sent heartbeat via results connection")
+                        b_messages.append((p_message, r))
+                    else:
+                        logger.error("Interchange discarding result_queue message of unknown type: {}".format(r['type']))
+
+                got_result = False
+                m = self._ready_managers[manager_id]
+                for (_, r) in b_messages:
+                    assert 'type' in r, f"Message is missing type entry: {r}"
+                    if r['type'] == 'result':
+                        got_result = True
+                        try:
+                            logger.debug(f"Removing task {r['task_id']} from manager record {manager_id!r}")
+                            m['tasks'].remove(r['task_id'])
+                        except Exception:
+                            # If we reach here, there's something very wrong.
+                            logger.exception("Ignoring exception removing task_id {} for manager {!r} with task list {}".format(
+                                r['task_id'],
+                                manager_id,
+                                m['tasks']))
+
+                b_messages_to_send = []
+                for (b_message, _) in b_messages:
+                    b_messages_to_send.append(b_message)
+
+                if b_messages_to_send:
+                    logger.debug("Sending messages on results_outgoing")
+                    self.results_outgoing.send_multipart(b_messages_to_send)
+                    logger.debug("Sent messages on results_outgoing")
+
+                logger.debug(f"Current tasks on manager {manager_id!r}: {m['tasks']}")
+                if len(m['tasks']) == 0 and m['idle_since'] is None:
+                    m['idle_since'] = time.time()
+
+                # A manager is only made interesting here if a result was
+                # received, which means there should be capacity for a new
+                # task now. Heartbeats and monitoring messages do not make a
+                # manager become interesting.
+                if got_result:
+                    interesting_managers.add(manager_id)
+            logger.debug("leaving results_incoming section")
+
+    def expire_bad_managers(self, interesting_managers: Set[bytes], hub_channel: Optional[zmq.Socket]) -> None:
+        bad_managers = [(manager_id, m) for (manager_id, m) in self._ready_managers.items() if
+                        time.time() - m['last_heartbeat'] > self.heartbeat_threshold]
+        for (manager_id, m) in bad_managers:
+            logger.debug("Last: {} Current: {}".format(m['last_heartbeat'], time.time()))
+            logger.warning(f"Too many heartbeats missed for manager {manager_id!r} - removing manager")
+            if m['active']:
+                m['active'] = False
+                self._send_monitoring_info(hub_channel, m)
+
+            logger.warning(f"Cancelling htex tasks {m['tasks']} on removed manager")
+            for tid in m['tasks']:
+                try:
+                    raise ManagerLost(manager_id, m['hostname'])
+                except Exception:
+                    result_package = {'type': 'result', 'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
+                    pkl_package = pickle.dumps(result_package)
+                    self.results_outgoing.send(pkl_package)
+            logger.warning("Sent failure reports, unregistering manager")
+            self._ready_managers.pop(manager_id, 'None')
+            if manager_id in interesting_managers:
+                interesting_managers.remove(manager_id)
+
+
+def start_file_logger(filename: str, level: int = logging.DEBUG, format_string: Optional[str] = None) -> None:
+    """Add a stream log handler.
+
+    Parameters
+    ---------
+
+    filename: string
+        Name of the file to write logs to. Required.
+    level: logging.LEVEL
+        Set the logging level. Default=logging.DEBUG
+        - format_string (string): Set the format string
+    format_string: string
+        Format string to use.
+
+    Returns
+    -------
+        None.
+    """
+    if format_string is None:
+        format_string = (
+
+            "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d "
+            "%(processName)s(%(process)d) %(threadName)s "
+            "%(funcName)s [%(levelname)s] %(message)s"
+
+        )
+
+    global logger
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(level)
+    handler = logging.FileHandler(filename)
+    handler.setLevel(level)
+    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+
+if __name__ == "__main__":
+    setproctitle("parsl: HTEX interchange")
+
+    config = pickle.load(sys.stdin.buffer)
+
+    ic = Interchange(**config)
+    ic.start()

{parsl-2024.6.10.dist-info → parsl-2024.6.17.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.6.10
+Version: 2024.6.17
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.06.10.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.06.17.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.6.10.dist-info → parsl-2024.6.17.dist-info}/RECORD

@@ -8,9 +8,9 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=91FjQiTUY383ueAjkBAgE21My9nba6SP2a2SrbB1r1Q,11250
-parsl/version.py,sha256=6Fjs2de_W12bDpbGxnHHYeGEOd0mla6OUrqgAATthTg,131
+parsl/version.py,sha256=bW2OSs-fGwn8vVUB6s290nY4Qe21tYD0iIBDkqvY198,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/app/app.py,sha256=4LO0MsFfcR4TNx9p87OkhuUImKfb-Okt5fco36LYibM,8502
+parsl/app/app.py,sha256=D5Ok_gt99mlclM_QfZbquHUBkibyG4tYdUN9ijRwUnQ,8345
 parsl/app/bash.py,sha256=iTpWH1K5E0e60nH23bwl97zNgg5BssFIqfp-182wkjA,5656
 parsl/app/errors.py,sha256=nJmOEPglAISfD3R1UsTZH-avqiSOJgx_DkpdL9B591w,3917
 parsl/app/futures.py,sha256=XU1NwkoNVsxy3KF5y0Ihsla5hPbhhuSikZInfS7h7Uo,2910
@@ -80,8 +80,8 @@ parsl/executors/flux/executor.py,sha256=gPq49CQwtSZYZggLZ0dCXdpUlllKHJbvR8WRKeGh
 parsl/executors/flux/flux_instance_manager.py,sha256=2KVcphlybF-ALYD_3_YjMUi0f5LkjdoJOT_783CW4H0,2036
 parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/high_throughput/errors.py,sha256=77ZGrw9suLh9tSWjyhCaIvnC9nRAOmrXsZmvHM6nT68,626
-parsl/executors/high_throughput/executor.py,sha256=rdsTocFEHWz6zDygHtHdr--udfoUt3HRS0tLdKsUXWE,37202
-parsl/executors/high_throughput/interchange.py,sha256=tCLNdSqzrz_jmhfif697K20CQpxiHl6kIJ40XFU1SBA,31681
+parsl/executors/high_throughput/executor.py,sha256=iRmAdQpHpmC0UDC5jDZ0O-BlZe_RhfItlqL5RIiD7os,37039
+parsl/executors/high_throughput/interchange.py,sha256=6avQQ8Ljtmuzpa5yjClswqdVEBPDnNBeKb_yn0XbVW4,31462
 parsl/executors/high_throughput/manager_record.py,sha256=9XppKjDW0DJ7SMkPNxsiDs-HvXGPLrTg6Ceyh4b6gNs,433
 parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
 parsl/executors/high_throughput/mpi_executor.py,sha256=B2CR1pHaGQzIwTrQ-_i08NZG-NwS6yr8y7nxPaa_rkA,3760
@@ -175,7 +175,7 @@ parsl/providers/grid_engine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 parsl/providers/grid_engine/grid_engine.py,sha256=jTQjKaJh4eEXGbhrrCcXFV4AVFo2t39iVpslDR8gF6o,8565
 parsl/providers/grid_engine/template.py,sha256=a7iViKr8LXcFTPmsf_qQeVK5o_RekOAIlUOF0X1q-2M,273
 parsl/providers/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/providers/kubernetes/kube.py,sha256=a6l4JPTwPWzytWmI3EjmtFThqCfS8pExV3D2BSAZtus,14507
+parsl/providers/kubernetes/kube.py,sha256=K9-HsdjOJ_kKqUbqnSu4CjuZj9CjtJO8LDLIWIETiz0,14485
 parsl/providers/kubernetes/template.py,sha256=VsRz6cmNaII-y4OdMT6sCwzQy95SJX6NMB0hmmFBhX4,50
 parsl/providers/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/providers/local/local.py,sha256=pTEcl9NnjRcL8FHcMeMEtJj1IXiAOxZ2Cih97Q5jDPY,11388
@@ -343,7 +343,7 @@ parsl/tests/test_htex/test_connected_blocks.py,sha256=gaXZSr__pIaLvKY6rF-4r1p_4d
 parsl/tests/test_htex/test_cpu_affinity_explicit.py,sha256=DVHrRCskDbJIrfB5YSi3ZSbfR4WzijA46aZfZzjNcrU,1382
 parsl/tests/test_htex/test_disconnected_blocks.py,sha256=3V1Ol9gMS6knjLTgIjB5GrunRSp4ANsJ_2vAvpyMR6c,1858
 parsl/tests/test_htex/test_drain.py,sha256=Z2Z5-3NfLL9tMgJh4JkVKLZZDl3Z2gDAbEFHDSGdItw,2288
-parsl/tests/test_htex/test_htex.py,sha256=3x6xO487k7nPqqze7PWc8qth4BqZiOzi-d719a3eDwg,3879
+parsl/tests/test_htex/test_htex.py,sha256=-gAD-c2h9EpgYG52IN4AUVBbsWnVD31-bIdaNffoGUY,4524
 parsl/tests/test_htex/test_manager_failure.py,sha256=N-obuSZ8f7XA_XcddoN2LWKSVtpKUZvTHb7BFelS3iQ,1143
 parsl/tests/test_htex/test_managers_command.py,sha256=Y-eUjtBzwW9erCYdph9bOesbkUvX8QUPqXt27DCgVS8,951
 parsl/tests/test_htex/test_missing_worker.py,sha256=gyp5i7_t-JHyJGtz_eXZKKBY5w8oqLOIxO6cJgGJMtQ,745
@@ -464,12 +464,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=qNEJ7nPimqd3Y7OWFLdYmNwJ6XDKlyfV_fTzasxsQw8,8690
-parsl-2024.6.10.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
-parsl-2024.6.10.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2024.6.10.data/scripts/process_worker_pool.py,sha256=weug6_LAMbqEKQhiI6ZMg8r3e-XBDw1-L5_COEt7caM,41879
-parsl-2024.6.10.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.6.10.dist-info/METADATA,sha256=zXwr4W5jcm17zCmQuBJZX1PLlX0_HzJsFhkqF_7KUB0,4124
-parsl-2024.6.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.6.10.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.6.10.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.6.10.dist-info/RECORD,,
+parsl-2024.6.17.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
+parsl-2024.6.17.data/scripts/interchange.py,sha256=kI1fSF30txw_eEicPSsxbycuz6Sdxiiyy2xrrk7jlZU,31449
+parsl-2024.6.17.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2024.6.17.data/scripts/process_worker_pool.py,sha256=weug6_LAMbqEKQhiI6ZMg8r3e-XBDw1-L5_COEt7caM,41879
+parsl-2024.6.17.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.6.17.dist-info/METADATA,sha256=IwP3sPUnicwrXXDVoYqbCuTzXAA1am762Xpw2RuKfso,4124
+parsl-2024.6.17.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.6.17.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.6.17.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.6.17.dist-info/RECORD,,