parsl 2025.9.8__tar.gz → 2025.9.15__tar.gz

{parsl-2025.9.8/parsl.egg-info → parsl-2025.9.15}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.8
+Version: 2025.9.15
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.08.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.09.15.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/curvezmq.py

@@ -101,17 +101,6 @@ class BaseContext(metaclass=ABCMeta):
         """
         self._ctx.destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        """Destroy then recreate the context.
-
-        Parameters
-        ----------
-        linger : int, optional
-            If specified, set LINGER on sockets prior to closing them.
-        """
-        self.destroy(linger)
-        self._ctx = zmq.Context()
-
 
 class ServerContext(BaseContext):
     """CurveZMQ server context
@@ -175,11 +164,6 @@ class ServerContext(BaseContext):
             self.auth_thread.stop()
         super().destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        super().recreate(linger)
-        if self.auth_thread:
-            self.auth_thread = self._start_auth_thread()
-
 
 class ClientContext(BaseContext):
     """CurveZMQ client context

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/executors/high_throughput/executor.py

@@ -332,6 +332,13 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self.encrypted = encrypted
         self.cert_dir = None
 
+        # This flag will enable/disable internal Python mismatch checks
+        # between the interchange and worker managers. This serves as a
+        # temporary workaround for Globus Compute to support different
+        # Python versions at the endpoint and worker layers. We can drop
+        # the flag once we implement modular internal message protocols.
+        self._check_python_mismatch: bool = True
+
         if not launch_cmd:
             launch_cmd = DEFAULT_LAUNCH_CMD
         self.launch_cmd = launch_cmd
@@ -568,6 +575,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                               "cert_dir": self.cert_dir,
                               "manager_selector": self.manager_selector,
                               "run_id": self.run_id,
+                              "_check_python_mismatch": self._check_python_mismatch,
                               }
 
         config_pickle = pickle.dumps(interchange_config)

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/executors/high_throughput/interchange.py

@@ -56,6 +56,7 @@ class Interchange:
                  cert_dir: Optional[str],
                  manager_selector: ManagerSelector,
                  run_id: str,
+                 _check_python_mismatch: bool,
                  ) -> None:
         """
         Parameters
@@ -99,6 +100,11 @@ class Interchange:
 
         cert_dir : str | None
             Path to the certificate directory.
+
+        _check_python_mismatch : bool
+            If True, the interchange and worker managers must run the same version of
+            Python. Running different versions can cause inter-process communication
+            errors, so proceed with caution.
         """
         self.cert_dir = cert_dir
         self.logdir = logdir
@@ -126,6 +132,7 @@ class Interchange:
         logger.info("Connected to client")
 
         self.run_id = run_id
+        self._check_python_mismatch = _check_python_mismatch
 
         self.hub_address = hub_address
         self.hub_zmq_port = hub_zmq_port
@@ -396,7 +403,9 @@ class Interchange:
             logger.info(f'Registration info for manager {manager_id!r}: {meta}')
             self._send_monitoring_info(monitoring_radio, new_rec)
 
-            if (mgr_minor_py, mgr_parsl_v) != (ix_minor_py, ix_parsl_v):
+            python_mismatch: bool = ix_minor_py != mgr_minor_py
+            parsl_mismatch: bool = ix_parsl_v != mgr_parsl_v
+            if parsl_mismatch or (self._check_python_mismatch and python_mismatch):
                 kill_event.set()
                 vm_exc = VersionMismatch(
                     f"py.v={ix_minor_py} parsl.v={ix_parsl_v}",

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/executors/high_throughput/zmq_pipes.py

@@ -74,51 +74,37 @@ class CommandClient:
 
         reply = '__PARSL_ZMQ_PIPES_MAGIC__'
         with self._lock:
-            for _ in range(max_retries):
-                try:
-                    logger.debug("Sending command client command")
-
-                    if timeout_s is not None:
-                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
-                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLOUT)
-                        if poll_result == zmq.POLLOUT:
-                            pass  # this is OK, so continue
-                        elif poll_result == 0:
-                            raise CommandClientTimeoutError("Waiting for command channel to be ready for a command")
-                        else:
-                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
-
-                    self.zmq_socket.send_pyobj(message, copy=True)
-
-                    if timeout_s is not None:
-                        logger.debug("Polling for command client response or timeout")
-                        remaining_time_s = start_time_s + timeout_s - time.monotonic()
-                        poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLIN)
-                        if poll_result == zmq.POLLIN:
-                            pass  # this is OK, so continue
-                        elif poll_result == 0:
-                            logger.error("Command timed-out - command client is now bad forever")
-                            self.ok = False
-                            raise CommandClientTimeoutError("Waiting for a reply from command channel")
-                        else:
-                            raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
-
-                    logger.debug("Receiving command client response")
-                    reply = self.zmq_socket.recv_pyobj()
-                    logger.debug("Received command client response")
-                except zmq.ZMQError:
-                    logger.exception("Potential ZMQ REQ-REP deadlock caught")
-                    logger.info("Trying to reestablish context")
-                    self.zmq_context.recreate()
-                    self.create_socket_and_bind()
+            logger.debug("Sending command client command")
+
+            if timeout_s is not None:
+                remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLOUT)
+                if poll_result == zmq.POLLOUT:
+                    pass  # this is OK, so continue
+                elif poll_result == 0:
+                    raise CommandClientTimeoutError("Waiting for command channel to be ready for a command")
                 else:
-                    break
-
-        if reply == '__PARSL_ZMQ_PIPES_MAGIC__':
-            logger.error("Command channel run retries exhausted. Unable to run command")
-            raise Exception("Command Channel retries exhausted")
+                    raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
+
+            self.zmq_socket.send_pyobj(message, copy=True)
+
+            if timeout_s is not None:
+                logger.debug("Polling for command client response or timeout")
+                remaining_time_s = start_time_s + timeout_s - time.monotonic()
+                poll_result = self.zmq_socket.poll(timeout=remaining_time_s * 1000, flags=zmq.POLLIN)
+                if poll_result == zmq.POLLIN:
+                    pass  # this is OK, so continue
+                elif poll_result == 0:
+                    logger.error("Command timed-out - command client is now bad forever")
+                    self.ok = False
+                    raise CommandClientTimeoutError("Waiting for a reply from command channel")
+                else:
+                    raise InternalConsistencyError(f"ZMQ poll returned unexpected value: {poll_result}")
 
-        return reply
+            logger.debug("Receiving command client response")
+            reply = self.zmq_socket.recv_pyobj()
+            logger.debug("Received command client response")
+            return reply
 
     def close(self):
         self.zmq_socket.close()

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/monitoring/monitoring.py

@@ -11,9 +11,9 @@ import typeguard
 
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.multiprocessing import (
-    SizedQueue,
     SpawnEvent,
     SpawnProcess,
+    SpawnQueue,
     join_terminate_close_proc,
 )
 from parsl.utils import RepresentationMixin
@@ -126,7 +126,7 @@ class MonitoringHub(RepresentationMixin):
         self.monitoring_hub_active = True
 
         self.resource_msgs: Queue[TaggedMonitoringMessage]
-        self.resource_msgs = SizedQueue()
+        self.resource_msgs = SpawnQueue()
 
         self.dbm_exit_event: ms.Event
         self.dbm_exit_event = SpawnEvent()

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/multiprocessing.py

@@ -4,7 +4,6 @@
 import logging
 import multiprocessing
 import multiprocessing.queues
-import platform
 from multiprocessing.context import ForkProcess as ForkProcessType
 from multiprocessing.context import SpawnProcess as SpawnProcessType
 from typing import Callable
@@ -21,54 +20,6 @@ SpawnEvent = SpawnContext.Event
 SpawnQueue = SpawnContext.Queue
 
 
-class MacSafeQueue(multiprocessing.queues.Queue):
-    """ Multiprocessing queues do not have qsize attributes on MacOS.
-    This is slower but more portable version of the multiprocessing Queue
-    that adds a explicit counter
-
-    Reference : https://github.com/keras-team/autokeras/commit/4ddd568b06b4045ace777bc0fb7bc18573b85a75
-    """
-
-    def __init__(self, *args, **kwargs):
-        if 'ctx' not in kwargs:
-            kwargs['ctx'] = multiprocessing.get_context('spawn')
-        super().__init__(*args, **kwargs)
-        self._counter = multiprocessing.Value('i', 0)
-
-    def put(self, *args, **kwargs):
-        # logger.critical("Putting item {}".format(args))
-        x = super().put(*args, **kwargs)
-        with self._counter.get_lock():
-            self._counter.value += 1
-        return x
-
-    def get(self, *args, **kwargs):
-        x = super().get(*args, **kwargs)
-        with self._counter.get_lock():
-            self._counter.value -= 1
-        # logger.critical("Getting item {}".format(x))
-        return x
-
-    def qsize(self):
-        return self._counter.value
-
-    def empty(self):
-        return not self._counter.value
-
-
-# SizedQueue should be constructable using the same calling
-# convention as multiprocessing.Queue but that entire signature
-# isn't expressible in mypy 0.790
-SizedQueue: Callable[..., multiprocessing.Queue]
-
-
-if platform.system() != 'Darwin':
-    import multiprocessing
-    SizedQueue = SpawnQueue
-else:
-    SizedQueue = MacSafeQueue
-
-
 def join_terminate_close_proc(process: SpawnProcessType, *, timeout: int = 30) -> None:
     """Increasingly aggressively terminate a process.
 

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/tests/test_curvezmq.py

@@ -296,48 +296,6 @@ def test_server_context_destroy(server_ctx: curvezmq.ServerContext, encrypted: b
         assert not server_ctx.auth_thread.pipe
 
 
-@pytest.mark.local
-@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
-def test_client_context_recreate(client_ctx: curvezmq.ClientContext):
-    hidden_ctx = client_ctx._ctx
-    sock = client_ctx.socket(zmq.REQ)
-
-    assert not sock.closed
-    assert not client_ctx.closed
-
-    client_ctx.recreate()
-
-    assert sock.closed
-    assert not client_ctx.closed
-    assert hidden_ctx != client_ctx._ctx
-    assert hidden_ctx.closed
-
-
-@pytest.mark.local
-@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
-def test_server_context_recreate(server_ctx: curvezmq.ServerContext, encrypted: bool):
-    hidden_ctx = server_ctx._ctx
-    sock = server_ctx.socket(zmq.REP)
-
-    assert not sock.closed
-    assert not server_ctx.closed
-    if encrypted:
-        assert server_ctx.auth_thread
-        auth_thread = server_ctx.auth_thread
-        assert auth_thread.pipe
-
-    server_ctx.recreate()
-
-    assert sock.closed
-    assert not server_ctx.closed
-    assert hidden_ctx.closed
-    assert hidden_ctx != server_ctx._ctx
-    if encrypted:
-        assert server_ctx.auth_thread
-        assert auth_thread != server_ctx.auth_thread
-        assert server_ctx.auth_thread.pipe
-
-
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_connection(

parsl-2025.9.15/parsl/tests/test_htex/test_command_concurrency_regression_1321.py

@@ -0,0 +1,54 @@
+import time
+from threading import Event, Thread
+
+import pytest
+
+import parsl
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+N_THREADS = 50
+DURATION_S = 10
+
+
+@pytest.mark.local
+def test_concurrency_blast():
+    """Blast interchange command channel from many threads.
+    """
+
+    cc = parsl.dfk().executors['htex_local'].command_client
+
+    threads = []
+
+    ok_so_far = True
+
+    for _ in range(N_THREADS):
+
+        # This event will be set if the thread reaches the end of its body.
+        event = Event()
+
+        thread = Thread(target=blast, args=(cc, event))
+        threads.append((thread, event))
+
+    for thread, event in threads:
+        thread.start()
+
+    for thread, event in threads:
+        thread.join()
+        if not event.is_set():
+            ok_so_far = False
+
+    assert ok_so_far, "at least one thread did not exit normally"
+
+
+def blast(cc, e):
+    target_end = time.monotonic() + DURATION_S
+
+    while time.monotonic() < target_end:
+        cc.run("WORKERS")
+        cc.run("MANGERs_PACKAGES")
+        cc.run("CONNECTED_BLOCKS")
+        cc.run("WORKER_BINDS")
+
+    # If any of the preceeding cc.run calls raises an exception, the thread
+    # will not set its successful completion event.
+    e.set()

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/tests/test_htex/test_interchange_exit_bad_registration.py

@@ -50,7 +50,8 @@ def test_exit_with_bad_registration(tmpd_cwd, try_assert):
                           "logging_level": logging.DEBUG,
                           "cert_dir": None,
                           "manager_selector": RandomManagerSelector(),
-                          "run_id": "test"
+                          "run_id": "test",
+                          "_check_python_mismatch": True,
                           }
 
     config_pickle = pickle.dumps(interchange_config)

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/tests/test_htex/test_priority_queue.py

@@ -46,6 +46,7 @@ def test_priority_queue():
             futures[(priority, i)] = fake_task(parsl_resource_specification=spec)
 
         provider.max_blocks = 1
+        htex.scale_out_facade(1)  # don't wait for the JSP to catch up
 
         # Wait for completion
         results = {

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/tests/test_htex/test_zmq_binding.py

@@ -29,7 +29,8 @@ def make_interchange(*,
                        logging_level=logging.INFO,
                        manager_selector=RandomManagerSelector(),
                        poll_period=10,
-                       run_id="test_run_id")
+                       run_id="test_run_id",
+                       _check_python_mismatch=True)
 
 
 @pytest.fixture

{parsl-2025.9.8 → parsl-2025.9.15}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.09.08'
+VERSION = '2025.09.15'

{parsl-2025.9.8 → parsl-2025.9.15/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.9.8
+Version: 2025.9.15
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.09.08.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.09.15.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2025.9.8 → parsl-2025.9.15}/parsl.egg-info/SOURCES.txt

@@ -210,7 +210,6 @@ parsl/serialize/proxystore.py
 parsl/tests/__init__.py
 parsl/tests/callables_helper.py
 parsl/tests/conftest.py
-parsl/tests/test_aalst_patterns.py
 parsl/tests/test_callables.py
 parsl/tests/test_curvezmq.py
 parsl/tests/test_execute_task.py
@@ -269,8 +268,6 @@ parsl/tests/manual_tests/test_regression_220.py
 parsl/tests/manual_tests/test_worker_count.py
 parsl/tests/site_tests/__init__.py
 parsl/tests/site_tests/site_config_selector.py
-parsl/tests/site_tests/test_provider.py
-parsl/tests/site_tests/test_site.py
 parsl/tests/sites/__init__.py
 parsl/tests/sites/test_affinity.py
 parsl/tests/sites/test_concurrent.py
@@ -306,12 +303,10 @@ parsl/tests/test_docs/test_from_slides.py
 parsl/tests/test_docs/test_kwargs.py
 parsl/tests/test_docs/test_tutorial_1.py
 parsl/tests/test_docs/test_workflow1.py
-parsl/tests/test_docs/test_workflow2.py
 parsl/tests/test_docs/test_workflow4.py
 parsl/tests/test_error_handling/__init__.py
 parsl/tests/test_error_handling/test_fail.py
 parsl/tests/test_error_handling/test_python_walltime.py
-parsl/tests/test_error_handling/test_rand_fail.py
 parsl/tests/test_error_handling/test_resource_spec.py
 parsl/tests/test_error_handling/test_retries.py
 parsl/tests/test_error_handling/test_retry_handler.py
@@ -323,6 +318,7 @@ parsl/tests/test_htex/__init__.py
 parsl/tests/test_htex/test_basic.py
 parsl/tests/test_htex/test_block_manager_selector_unit.py
 parsl/tests/test_htex/test_command_client_timeout.py
+parsl/tests/test_htex/test_command_concurrency_regression_1321.py
 parsl/tests/test_htex/test_connected_blocks.py
 parsl/tests/test_htex/test_cpu_affinity_explicit.py
 parsl/tests/test_htex/test_disconnected_blocks.py
@@ -410,7 +406,6 @@ parsl/tests/test_regression/test_221.py
 parsl/tests/test_regression/test_226.py
 parsl/tests/test_regression/test_2652.py
 parsl/tests/test_regression/test_69a.py
-parsl/tests/test_regression/test_854.py
 parsl/tests/test_regression/test_97_parallelism_0.py
 parsl/tests/test_regression/test_98.py
 parsl/tests/test_scaling/__init__.py

parsl-2025.9.8/parsl/tests/site_tests/test_provider.py

@@ -1,88 +0,0 @@
-import argparse
-import logging
-import time
-
-import pytest
-
-import parsl
-from parsl.app.app import python_app  # , bash_app
-from parsl.jobs.states import JobState
-from parsl.tests.site_tests.site_config_selector import fresh_config
-
-logger = logging.getLogger(__name__)
-
-
-@python_app
-def platform(sleep=10, stdout=None):
-    import time
-    time.sleep(sleep)
-    return True
-
-
-@pytest.mark.local
-@pytest.mark.skip("This test cannot run on sites which cannot be identified by site_config_selector")
-def test_provider():
-    """ Provider scaling
-    """
-    logger.info("Starting test_provider")
-    config = fresh_config()
-    name = config.executors[0].label
-    parsl.load(config)
-
-    dfk = parsl.dfk()
-    logger.info("Trying to get executor : {}".format(name))
-
-    x = platform(sleep=0)
-    logger.info("Result is {}".format(x.result()))
-
-    executor = dfk.executors[name]
-    provider = dfk.executors[name].provider
-
-    # At this point we should have 1 job
-    _, current_jobs = executor._get_block_and_job_ids()
-    assert len(current_jobs) == 1, "Expected 1 job at init, got {}".format(len(current_jobs))
-
-    logger.info("Getting provider status (1)")
-    status = provider.status(current_jobs)
-    logger.info("Got provider status")
-    assert status[0].state == JobState.RUNNING, "Expected job to be in state RUNNING"
-
-    # Scale down to 0
-    scale_in_blocks = executor.scale_in(blocks=1)
-    logger.info("Now sleeping 60 seconds")
-    time.sleep(60)
-    logger.info("Sleep finished")
-    logger.info("Getting provider status (2)")
-    status = executor.status()
-    logger.info("Got executor status")
-    logger.info("Block status: {}".format(status))
-    assert status[scale_in_blocks[0]].terminal is True, "Terminal state"
-    logger.info("Job in terminal state")
-
-    _, current_jobs = executor._get_block_and_job_ids()
-    # PR 1952 stoped removing scale_in blocks from self.blocks_to_job_id
-    # A new PR will handle removing blocks from self.block
-    # this includes failed/completed/canceled blocks
-    assert len(current_jobs) == 1, "Expected current_jobs == 1"
-    dfk.cleanup()
-    parsl.clear()
-    logger.info("Ended test_provider")
-    return True
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--count", default="4",
-                        help="Count of apps to launch")
-    parser.add_argument("-t", "--time", default="60",
-                        help="Sleep time for each app")
-
-    parser.add_argument("-d", "--debug", action='store_true',
-                        help="Count of apps to launch")
-    args = parser.parse_args()
-
-    if args.debug:
-        parsl.set_stream_logger()
-
-    x = test_provider()

parsl-2025.9.8/parsl/tests/site_tests/test_site.py

@@ -1,70 +0,0 @@
-import argparse
-
-import pytest
-
-import parsl
-from parsl.app.app import python_app
-from parsl.tests.site_tests.site_config_selector import fresh_config
-
-
-@python_app
-def platform(sleep=10, stdout=None):
-    import platform
-    import time
-    time.sleep(sleep)
-    return platform.uname()
-
-
-@pytest.mark.local
-@pytest.mark.skip("The behaviour this test is testing is unclear: there is no guarantee that tasks will go to different nodes")
-def test_platform(n=2, sleep_dur=10):
-    """ This should sleep to make sure that concurrent apps will go to different workers
-    on different nodes.
-    """
-    config = fresh_config()
-    if config.executors[0].label == "htex_local":
-        return
-
-    parsl.load(fresh_config())
-
-    dfk = parsl.dfk()
-    name = list(dfk.executors.keys())[0]
-    print("Trying to get executor : ", name)
-
-    x = [platform(sleep=1) for i in range(2)]
-    print([i.result() for i in x])
-
-    print("Executor : ", dfk.executors[name])
-    print("Connected   : ", dfk.executors[name].connected_workers())
-    print("Outstanding : ", dfk.executors[name].outstanding())
-
-    d = []
-    for i in range(0, n):
-        x = platform(sleep=sleep_dur)
-        d.append(x)
-
-    pinfo = set([i.result()for i in d])
-    assert len(pinfo) == 2, "Expected two nodes, instead got {}".format(pinfo)
-
-    print("Test passed")
-
-    dfk.cleanup()
-    parsl.clear()
-    return True
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--count", default="4",
-                        help="Count of apps to launch")
-    parser.add_argument("-t", "--time", default="60",
-                        help="Sleep time for each app")
-    parser.add_argument("-d", "--debug", action='store_true',
-                        help="Count of apps to launch")
-    args = parser.parse_args()
-
-    if args.debug:
-        parsl.set_stream_logger()
-
-    x = test_platform(n=int(args.count), sleep_dur=int(args.time))