parsl 2025.1.13__py3-none-any.whl → 2025.1.27__py3-none-any.whl

parsl/executors/high_throughput/executor.py

@@ -8,7 +8,7 @@ import warnings
 from collections import defaultdict
 from concurrent.futures import Future
 from dataclasses import dataclass
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import typeguard
 
@@ -357,10 +357,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         return self.logdir
 
     def validate_resource_spec(self, resource_specification: dict):
-        """HTEX supports the following *Optional* resource specifications:
-        priority: lower value is higher priority"""
         if resource_specification:
-            acceptable_fields = {'priority'}
+            acceptable_fields: Set[str] = set()  # add new resource spec field names here to make htex accept them
             keys = set(resource_specification.keys())
             invalid_keys = keys - acceptable_fields
             if invalid_keys:
@@ -433,8 +431,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self._start_result_queue_thread()
         self._start_local_interchange_process()
 
-        logger.debug("Created result queue thread: %s", self._result_queue_thread)
-
         self.initialize_scaling()
 
     @wrap_with_logs
@@ -531,6 +527,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         get the worker task and result ports that the interchange has bound to.
         """
 
+        assert self.interchange_proc is None, f"Already exists! {self.interchange_proc!r}"
+
         interchange_config = {"client_address": self.loopback_address,
                               "client_ports": (self.outgoing_q.port,
                                                self.incoming_q.port,
@@ -565,7 +563,12 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         except CommandClientTimeoutError:
             logger.error("Interchange has not completed initialization. Aborting")
             raise Exception("Interchange failed to start")
-        logger.debug("Got worker ports")
+        logger.debug(
+            "Interchange process started (%r).  Worker ports: %d, %d",
+            self.interchange_proc,
+            self.worker_task_port,
+            self.worker_result_port
+        )
 
     def _start_result_queue_thread(self):
         """Method to start the result queue thread as a daemon.
@@ -573,15 +576,13 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         Checks if a thread already exists, then starts it.
         Could be used later as a restart if the result queue thread dies.
         """
-        if self._result_queue_thread is None:
-            logger.debug("Starting result queue thread")
-            self._result_queue_thread = threading.Thread(target=self._result_queue_worker, name="HTEX-Result-Queue-Thread")
-            self._result_queue_thread.daemon = True
-            self._result_queue_thread.start()
-            logger.debug("Started result queue thread")
+        assert self._result_queue_thread is None, f"Already exists! {self._result_queue_thread!r}"
 
-        else:
-            logger.error("Result queue thread already exists, returning")
+        logger.debug("Starting result queue thread")
+        self._result_queue_thread = threading.Thread(target=self._result_queue_worker, name="HTEX-Result-Queue-Thread")
+        self._result_queue_thread.daemon = True
+        self._result_queue_thread.start()
+        logger.debug("Started result queue thread: %r", self._result_queue_thread)
 
     def hold_worker(self, worker_id: str) -> None:
         """Puts a worker on hold, preventing scheduling of additional tasks to it.

parsl/executors/high_throughput/interchange.py

@@ -9,7 +9,7 @@ import queue
 import sys
 import threading
 import time
-from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, cast
 
 import zmq
 
@@ -132,6 +132,11 @@ class Interchange:
         self.hub_zmq_port = hub_zmq_port
 
         self.pending_task_queue: queue.Queue[Any] = queue.Queue(maxsize=10 ** 6)
+
+        # count of tasks that have been received from the submit side
+        self.task_counter = 0
+
+        # count of tasks that have been sent out to worker pools
         self.count = 0
 
         self.worker_ports = worker_ports
@@ -201,28 +206,6 @@ class Interchange:
 
         return tasks
 
-    @wrap_with_logs(target="interchange")
-    def task_puller(self) -> NoReturn:
-        """Pull tasks from the incoming tasks zmq pipe onto the internal
-        pending task queue
-        """
-        logger.info("Starting")
-        task_counter = 0
-
-        while True:
-            logger.debug("launching recv_pyobj")
-            try:
-                msg = self.task_incoming.recv_pyobj()
-            except zmq.Again:
-                # We just timed out while attempting to receive
-                logger.debug("zmq.Again with {} tasks in internal queue".format(self.pending_task_queue.qsize()))
-                continue
-
-            logger.debug("putting message onto pending_task_queue")
-            self.pending_task_queue.put(msg)
-            task_counter += 1
-            logger.debug(f"Fetched {task_counter} tasks so far")
-
     def _send_monitoring_info(self, monitoring_radio: Optional[MonitoringRadioSender], manager: ManagerRecord) -> None:
         if monitoring_radio:
             logger.info("Sending message {} to MonitoringHub".format(manager))
@@ -234,79 +217,68 @@ class Interchange:
 
             monitoring_radio.send((MessageType.NODE_INFO, d))
 
-    @wrap_with_logs(target="interchange")
-    def _command_server(self) -> NoReturn:
+    def process_command(self, monitoring_radio: Optional[MonitoringRadioSender]) -> None:
         """ Command server to run async command to the interchange
         """
-        logger.debug("Command Server Starting")
-
-        if self.hub_address is not None and self.hub_zmq_port is not None:
-            logger.debug("Creating monitoring radio to %s:%s", self.hub_address, self.hub_zmq_port)
-            monitoring_radio = ZMQRadioSender(self.hub_address, self.hub_zmq_port)
-        else:
-            monitoring_radio = None
+        logger.debug("entering command_server section")
 
         reply: Any  # the type of reply depends on the command_req received (aka this needs dependent types...)
 
-        while True:
-            try:
-                command_req = self.command_channel.recv_pyobj()
-                logger.debug("Received command request: {}".format(command_req))
-                if command_req == "CONNECTED_BLOCKS":
-                    reply = self.connected_block_history
-
-                elif command_req == "WORKERS":
-                    num_workers = 0
-                    for manager in self._ready_managers.values():
-                        num_workers += manager['worker_count']
-                    reply = num_workers
-
-                elif command_req == "MANAGERS":
-                    reply = []
-                    for manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        idle_since = m['idle_since']
-                        if idle_since is not None:
-                            idle_duration = time.time() - idle_since
-                        else:
-                            idle_duration = 0.0
-                        resp = {'manager': manager_id.decode('utf-8'),
-                                'block_id': m['block_id'],
-                                'worker_count': m['worker_count'],
-                                'tasks': len(m['tasks']),
-                                'idle_duration': idle_duration,
-                                'active': m['active'],
-                                'parsl_version': m['parsl_version'],
-                                'python_version': m['python_version'],
-                                'draining': m['draining']}
-                        reply.append(resp)
-
-                elif command_req.startswith("HOLD_WORKER"):
-                    cmd, s_manager = command_req.split(';')
-                    manager_id = s_manager.encode('utf-8')
-                    logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
-                    if manager_id in self._ready_managers:
-                        m = self._ready_managers[manager_id]
-                        m['active'] = False
-                        self._send_monitoring_info(monitoring_radio, m)
+        if self.command_channel in self.socks and self.socks[self.command_channel] == zmq.POLLIN:
+
+            command_req = self.command_channel.recv_pyobj()
+            logger.debug("Received command request: {}".format(command_req))
+            if command_req == "CONNECTED_BLOCKS":
+                reply = self.connected_block_history
+
+            elif command_req == "WORKERS":
+                num_workers = 0
+                for manager in self._ready_managers.values():
+                    num_workers += manager['worker_count']
+                reply = num_workers
+
+            elif command_req == "MANAGERS":
+                reply = []
+                for manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    idle_since = m['idle_since']
+                    if idle_since is not None:
+                        idle_duration = time.time() - idle_since
                     else:
-                        logger.warning("Worker to hold was not in ready managers list")
-
-                    reply = None
+                        idle_duration = 0.0
+                    resp = {'manager': manager_id.decode('utf-8'),
+                            'block_id': m['block_id'],
+                            'worker_count': m['worker_count'],
+                            'tasks': len(m['tasks']),
+                            'idle_duration': idle_duration,
+                            'active': m['active'],
+                            'parsl_version': m['parsl_version'],
+                            'python_version': m['python_version'],
+                            'draining': m['draining']}
+                    reply.append(resp)
+
+            elif command_req.startswith("HOLD_WORKER"):
+                cmd, s_manager = command_req.split(';')
+                manager_id = s_manager.encode('utf-8')
+                logger.info("Received HOLD_WORKER for {!r}".format(manager_id))
+                if manager_id in self._ready_managers:
+                    m = self._ready_managers[manager_id]
+                    m['active'] = False
+                    self._send_monitoring_info(monitoring_radio, m)
+                else:
+                    logger.warning("Worker to hold was not in ready managers list")
 
-                elif command_req == "WORKER_PORTS":
-                    reply = (self.worker_task_port, self.worker_result_port)
+                reply = None
 
-                else:
-                    logger.error(f"Received unknown command: {command_req}")
-                    reply = None
+            elif command_req == "WORKER_PORTS":
+                reply = (self.worker_task_port, self.worker_result_port)
 
-                logger.debug("Reply: {}".format(reply))
-                self.command_channel.send_pyobj(reply)
+            else:
+                logger.error(f"Received unknown command: {command_req}")
+                reply = None
 
-            except zmq.Again:
-                logger.debug("Command thread is alive")
-                continue
+            logger.debug("Reply: {}".format(reply))
+            self.command_channel.send_pyobj(reply)
 
     @wrap_with_logs
     def start(self) -> None:
@@ -326,21 +298,13 @@ class Interchange:
 
         start = time.time()
 
-        self._task_puller_thread = threading.Thread(target=self.task_puller,
-                                                    name="Interchange-Task-Puller",
-                                                    daemon=True)
-        self._task_puller_thread.start()
-
-        self._command_thread = threading.Thread(target=self._command_server,
-                                                name="Interchange-Command",
-                                                daemon=True)
-        self._command_thread.start()
-
         kill_event = threading.Event()
 
         poller = zmq.Poller()
         poller.register(self.task_outgoing, zmq.POLLIN)
         poller.register(self.results_incoming, zmq.POLLIN)
+        poller.register(self.task_incoming, zmq.POLLIN)
+        poller.register(self.command_channel, zmq.POLLIN)
 
         # These are managers which we should examine in an iteration
         # for scheduling a job (or maybe any other attention?).
@@ -351,6 +315,8 @@ class Interchange:
         while not kill_event.is_set():
             self.socks = dict(poller.poll(timeout=poll_period))
 
+            self.process_command(monitoring_radio)
+            self.process_task_incoming()
             self.process_task_outgoing_incoming(interesting_managers, monitoring_radio, kill_event)
             self.process_results_incoming(interesting_managers, monitoring_radio)
             self.expire_bad_managers(interesting_managers, monitoring_radio)
@@ -362,6 +328,18 @@ class Interchange:
         logger.info(f"Processed {self.count} tasks in {delta} seconds")
         logger.warning("Exiting")
 
+    def process_task_incoming(self) -> None:
+        """Process incoming task message(s).
+        """
+
+        if self.task_incoming in self.socks and self.socks[self.task_incoming] == zmq.POLLIN:
+            logger.debug("start task_incoming section")
+            msg = self.task_incoming.recv_pyobj()
+            logger.debug("putting message onto pending_task_queue")
+            self.pending_task_queue.put(msg)
+            self.task_counter += 1
+            logger.debug(f"Fetched {self.task_counter} tasks so far")
+
     def process_task_outgoing_incoming(
             self,
             interesting_managers: Set[bytes],

parsl/executors/high_throughput/zmq_pipes.py

@@ -213,7 +213,6 @@ class ResultsIncoming:
         """Get a message from the queue, returning None if timeout expires
         without a message. timeout is measured in milliseconds.
         """
-        logger.debug("Waiting for ResultsIncoming message")
         socks = dict(self.poller.poll(timeout=timeout_ms))
         if self.results_receiver in socks and socks[self.results_receiver] == zmq.POLLIN:
             m = self.results_receiver.recv_multipart()

parsl/tests/configs/globus_compute.py

@@ -0,0 +1,20 @@
+import os
+
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+
+
+def fresh_config():
+
+    endpoint_id = os.environ["GLOBUS_COMPUTE_ENDPOINT"]
+
+    return Config(
+        executors=[
+            GlobusComputeExecutor(
+                executor=Executor(endpoint_id=endpoint_id),
+                label="globus_compute",
+            )
+        ]
+    )

parsl/tests/conftest.py

@@ -163,6 +163,10 @@ def pytest_configure(config):
         'markers',
         'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
     )
+    config.addinivalue_line(
+        'markers',
+        'issue_3620: Marks tests that do not work correctly on GlobusComputeExecutor (ref: issue 3620)'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')

parsl/tests/test_checkpointing/test_python_checkpoint_1.py

@@ -27,8 +27,5 @@ def test_initial_checkpoint_write():
 
     cpt_dir = parsl.dfk().checkpoint()
 
-    cptpath = cpt_dir + '/dfk.pkl'
-    assert os.path.exists(cptpath), f"DFK checkpoint missing: {cptpath}"
-
     cptpath = cpt_dir + '/tasks.pkl'
     assert os.path.exists(cptpath), f"Tasks checkpoint missing: {cptpath}"

parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,3 +1,5 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.executors import WorkQueueExecutor
@@ -11,6 +13,7 @@ def double(x, parsl_resource_specification={}):
     return x * 2
 
 
+@pytest.mark.issue_3620
 def test_resource(n=2):
     executors = parsl.dfk().executors
     executor = None

parsl/tests/test_htex/test_interchange_exit_bad_registration.py

@@ -0,0 +1,120 @@
+import json
+import logging
+import os
+import pickle
+import platform
+import subprocess
+import time
+
+import psutil
+import pytest
+import zmq
+
+import parsl.executors.high_throughput.zmq_pipes as zmq_pipes
+from parsl.executors.high_throughput.executor import DEFAULT_INTERCHANGE_LAUNCH_CMD
+from parsl.executors.high_throughput.manager_selector import RandomManagerSelector
+from parsl.version import VERSION as PARSL_VERSION
+
+P_ms = 10
+
+
+@pytest.mark.local
+def test_exit_with_bad_registration(tmpd_cwd, try_assert):
+    """Test that the interchange exits when it receives a bad registration message.
+    This complements parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py
+    which tests that the interchange is resistent to other forms of bad message.
+    """
+
+    outgoing_q = zmq_pipes.TasksOutgoing(
+        "127.0.0.1", (49152, 65535), None
+    )
+    incoming_q = zmq_pipes.ResultsIncoming(
+        "127.0.0.1", (49152, 65535), None
+    )
+    command_client = zmq_pipes.CommandClient(
+        "127.0.0.1", (49152, 65535), None
+    )
+
+    interchange_config = {"client_address": "127.0.0.1",
+                          "client_ports": (outgoing_q.port,
+                                           incoming_q.port,
+                                           command_client.port),
+                          "interchange_address": "127.0.0.1",
+                          "worker_ports": None,
+                          "worker_port_range": (50000, 60000),
+                          "hub_address": None,
+                          "hub_zmq_port": None,
+                          "logdir": tmpd_cwd,
+                          "heartbeat_threshold": 120,
+                          "poll_period": P_ms,
+                          "logging_level": logging.DEBUG,
+                          "cert_dir": None,
+                          "manager_selector": RandomManagerSelector(),
+                          "run_id": "test"
+                          }
+
+    config_pickle = pickle.dumps(interchange_config)
+
+    interchange_proc = subprocess.Popen(DEFAULT_INTERCHANGE_LAUNCH_CMD, stdin=subprocess.PIPE)
+    stdin = interchange_proc.stdin
+    assert stdin is not None, "Popen should have created an IO object (vs default None) because of PIPE mode"
+
+    stdin.write(config_pickle)
+    stdin.flush()
+    stdin.close()
+
+    # wait for interchange to be alive, by waiting for the command thread to become
+    # responsive. if the interchange process didn't start enough to get the command
+    # thread running, this will time out.
+
+    (task_port, result_port) = command_client.run("WORKER_PORTS", timeout_s=120)
+
+    # now we'll assume that if the interchange command thread is responding,
+    # then the worker polling code is also running and that the interchange has
+    # started successfully.
+
+    # send bad registration message as if from a new worker pool. The badness here
+    # is that the Python version does not match the real Python version - which
+    # unlike some other bad interchange messages, should cause the interchange
+    # to shut down.
+
+    msg = {'type': 'registration',
+           'parsl_v': PARSL_VERSION,
+           'python_v': "{}.{}.{}".format(1, 1, 1),  # this is the bad bit
+           'worker_count': 1,
+           'uid': 'testuid',
+           'block_id': 0,
+           'start_time': time.time(),
+           'prefetch_capacity': 0,
+           'max_capacity': 1,
+           'os': platform.system(),
+           'hostname': platform.node(),
+           'dir': os.getcwd(),
+           'cpu_count': psutil.cpu_count(logical=False),
+           'total_memory': psutil.virtual_memory().total,
+           }
+
+    # connect to worker port and send this message.
+
+    context = zmq.Context()
+    channel_timeout = 10000  # in milliseconds
+    task_channel = context.socket(zmq.DEALER)
+    task_channel.setsockopt(zmq.LINGER, 0)
+    task_channel.setsockopt(zmq.IDENTITY, b'testid')
+
+    task_channel.set_hwm(0)
+    task_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+    task_channel.connect(f"tcp://127.0.0.1:{task_port}")
+
+    b_msg = json.dumps(msg).encode('utf-8')
+
+    task_channel.send(b_msg)
+
+    # check that the interchange exits within some reasonable time
+    try_assert(lambda: interchange_proc.poll() is not None, "Interchange did not exit after killing watched client process", timeout_ms=5000)
+
+    # See issue #3697 - ideally the interchange would exit cleanly, but it does not.
+    # assert interchange_proc.poll() == 0, "Interchange exited with an error code, not 0"
+
+    task_channel.close()
+    context.term()

parsl/tests/test_htex/test_resource_spec_validation.py

@@ -30,13 +30,6 @@ def test_resource_spec_validation():
     assert ret_val is None
 
 
-@pytest.mark.local
-def test_resource_spec_validation_one_key():
-    htex = HighThroughputExecutor()
-    ret_val = htex.validate_resource_spec({"priority": 2})
-    assert ret_val is None
-
-
 @pytest.mark.local
 def test_resource_spec_validation_bad_keys():
     htex = HighThroughputExecutor()

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -43,3 +43,6 @@ def test_future_fail_dependency():
     # Future, plain_fut, somewhere in its str
 
     assert repr(plain_fut) in str(ex)
+    assert len(ex.dependent_exceptions_tids) == 1
+    assert isinstance(ex.dependent_exceptions_tids[0][0], ValueError)
+    assert ex.dependent_exceptions_tids[0][1].startswith("<Future ")

parsl/tests/test_python_apps/test_fail.py

@@ -27,17 +27,32 @@ def test_no_deps():
             pass
 
 
-@pytest.mark.parametrize("fail_probs", ((1, 0), (0, 1)))
-def test_fail_sequence(fail_probs):
-    """Test failure in a sequence of dependencies
+def test_fail_sequence_first():
+    t1 = random_fail(fail_prob=1)
+    t2 = random_fail(fail_prob=0, inputs=[t1])
+    t_final = random_fail(fail_prob=0, inputs=[t2])
 
-    App1 -> App2 ... -> AppN
-    """
+    with pytest.raises(DependencyError):
+        t_final.result()
+
+    assert len(t_final.exception().dependent_exceptions_tids) == 1
+    assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], DependencyError)
+    assert t_final.exception().dependent_exceptions_tids[0][1].startswith("task ")
 
-    t1_fail_prob, t2_fail_prob = fail_probs
-    t1 = random_fail(fail_prob=t1_fail_prob)
-    t2 = random_fail(fail_prob=t2_fail_prob, inputs=[t1])
+    assert hasattr(t_final.exception(), '__cause__')
+    assert t_final.exception().__cause__ == t1.exception()
+
+
+def test_fail_sequence_middle():
+    t1 = random_fail(fail_prob=0)
+    t2 = random_fail(fail_prob=1, inputs=[t1])
     t_final = random_fail(fail_prob=0, inputs=[t2])
 
     with pytest.raises(DependencyError):
         t_final.result()
+
+    assert len(t_final.exception().dependent_exceptions_tids) == 1
+    assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], ManufacturedTestFailure)
+
+    assert hasattr(t_final.exception(), '__cause__')
+    assert t_final.exception().__cause__ == t2.exception()

parsl/tests/test_python_apps/test_join.py

@@ -97,7 +97,10 @@ def test_error():
     f = outer_error()
     e = f.exception()
     assert isinstance(e, JoinError)
+
+    assert len(e.dependent_exceptions_tids) == 1
     assert isinstance(e.dependent_exceptions_tids[0][0], InnerError)
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
 
 def test_two_errors():
@@ -109,10 +112,12 @@ def test_two_errors():
     de0 = e.dependent_exceptions_tids[0][0]
     assert isinstance(de0, InnerError)
     assert de0.args[0] == "Error A"
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
     de1 = e.dependent_exceptions_tids[1][0]
     assert isinstance(de1, InnerError)
     assert de1.args[0] == "Error B"
+    assert e.dependent_exceptions_tids[1][1].startswith("task ")
 
 
 def test_one_error_one_result():
@@ -125,6 +130,7 @@ def test_one_error_one_result():
     de0 = e.dependent_exceptions_tids[0][0]
     assert isinstance(de0, InnerError)
     assert de0.args[0] == "Error A"
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
 
 @join_app

parsl/tests/test_python_apps/test_memoize_1.py

@@ -2,7 +2,6 @@ import argparse
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
 
 
 @python_app(cache=True)