parsl 2024.10.7__tar.gz → 2024.10.21__tar.gz

{parsl-2024.10.7/parsl.egg-info → parsl-2024.10.21}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.10.7
+Version: 2024.10.21
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.10.07.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.10.21.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -11,12 +11,11 @@ Keywords: Workflows,Scientific computing
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8.0
+Requires-Python: >=3.9.0
 Provides-Extra: monitoring
 Provides-Extra: visualization
 Provides-Extra: aws

{parsl-2024.10.7 → parsl-2024.10.21}/README.rst

@@ -117,7 +117,7 @@ For Developers
 Requirements
 ============
 
-Parsl is supported in Python 3.8+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
+Parsl is supported in Python 3.9+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
 
 Code of Conduct
 ===============

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/errors.py

@@ -1,4 +1,6 @@
 """Exceptions raise by Executors."""
+from typing import Set
+
 from parsl.errors import ParslError
 from parsl.executors.base import ParslExecutor
 
@@ -44,6 +46,17 @@ class UnsupportedFeatureError(ExecutorError):
                                                                  self.current_executor)
 
 
+class InvalidResourceSpecification(ExecutorError):
+    """Error raised when Invalid input is supplied via resource Specification"""
+
+    def __init__(self, invalid_keys: Set[str], message: str = ''):
+        self.invalid_keys = invalid_keys
+        self.message = message
+
+    def __str__(self):
+        return f"Invalid Resource Specification Supplied: {self.invalid_keys}. {self.message}"
+
+
 class ScalingFailed(ExecutorError):
     """Scaling failed due to error in Execution provider."""
 

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/high_throughput/executor.py

@@ -16,16 +16,17 @@ from parsl import curvezmq
 from parsl.addresses import get_all_addresses
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.staging import Staging
-from parsl.executors.errors import BadMessage, ScalingFailed
+from parsl.executors.errors import (
+    BadMessage,
+    InvalidResourceSpecification,
+    ScalingFailed,
+)
 from parsl.executors.high_throughput import zmq_pipes
 from parsl.executors.high_throughput.errors import CommandClientTimeoutError
 from parsl.executors.high_throughput.manager_selector import (
     ManagerSelector,
     RandomManagerSelector,
 )
-from parsl.executors.high_throughput.mpi_prefix_composer import (
-    InvalidResourceSpecification,
-)
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
 from parsl.process_loggers import wrap_with_logs
@@ -145,6 +146,11 @@ GENERAL_HTEX_PARAM_DOCS = """provider : :class:`~parsl.providers.base.ExecutionP
 
     encrypted : bool
         Flag to enable/disable encryption (CurveZMQ). Default is False.
+
+    manager_selector: ManagerSelector
+        Determines what strategy the interchange uses to select managers during task distribution.
+        See API reference under "Manager Selectors" regarding the various manager selectors.
+        Default: 'RandomManagerSelector'
 """  # Documentation for params used by both HTEx and MPIEx
 
 
@@ -340,15 +346,17 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         return self.logdir
 
     def validate_resource_spec(self, resource_specification: dict):
-        """HTEX does not support *any* resource_specification options and
-        will raise InvalidResourceSpecification is any are passed to it"""
+        """HTEX supports the following *Optional* resource specifications:
+        priority: lower value is higher priority"""
         if resource_specification:
-            raise InvalidResourceSpecification(
-                set(resource_specification.keys()),
-                ("HTEX does not support the supplied resource_specifications."
-                 "For MPI applications consider using the MPIExecutor. "
-                 "For specifications for core count/memory/walltime, consider using WorkQueueExecutor. ")
-            )
+            acceptable_fields = {'priority'}
+            keys = set(resource_specification.keys())
+            invalid_keys = keys - acceptable_fields
+            if invalid_keys:
+                message = "Task resource specification only accepts these types of resources: {}".format(
+                    ', '.join(acceptable_fields))
+                logger.error(message)
+                raise InvalidResourceSpecification(set(invalid_keys), message)
         return
 
     def initialize_scaling(self):
@@ -459,9 +467,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                     except pickle.UnpicklingError:
                         raise BadMessage("Message received could not be unpickled")
 
-                    if msg['type'] == 'heartbeat':
-                        continue
-                    elif msg['type'] == 'result':
+                    if msg['type'] == 'result':
                         try:
                             tid = msg['task_id']
                         except Exception:
@@ -581,7 +587,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
     def outstanding(self) -> int:
         """Returns the count of tasks outstanding across the interchange
         and managers"""
-        return self.command_client.run("OUTSTANDING_C")
+        return len(self.tasks)
 
     @property
     def connected_workers(self) -> int:
@@ -658,7 +664,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         except TypeError:
             raise SerializationError(func.__name__)
 
-        msg = {"task_id": task_id, "buffer": fn_buf}
+        msg = {"task_id": task_id, "resource_spec": resource_specification, "buffer": fn_buf}
 
         # Post task to the outgoing queue
         self.outgoing_q.put(msg)

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/high_throughput/interchange.py

@@ -6,7 +6,6 @@ import os
 import pickle
 import platform
 import queue
-import signal
 import sys
 import threading
 import time
@@ -252,13 +251,7 @@ class Interchange:
             try:
                 command_req = self.command_channel.recv_pyobj()
                 logger.debug("Received command request: {}".format(command_req))
-                if command_req == "OUTSTANDING_C":
-                    outstanding = self.pending_task_queue.qsize()
-                    for manager in self._ready_managers.values():
-                        outstanding += len(manager['tasks'])
-                    reply = outstanding
-
-                elif command_req == "CONNECTED_BLOCKS":
+                if command_req == "CONNECTED_BLOCKS":
                     reply = self.connected_block_history
 
                 elif command_req == "WORKERS":
@@ -319,16 +312,6 @@ class Interchange:
         """ Start the interchange
         """
 
-        # If a user workflow has set its own signal handler for sigterm, that
-        # handler will be inherited by the interchange process because it is
-        # launched as a multiprocessing fork process.
-        # That can interfere with the interchange shutdown mechanism, which is
-        # to receive a SIGTERM and exit immediately.
-        # See Parsl issue #2343 (Threads and multiprocessing cannot be
-        # intermingled without deadlocks) which talks about other fork-related
-        # parent-process-inheritance problems.
-        signal.signal(signal.SIGTERM, signal.SIG_DFL)
-
         logger.info("Starting main interchange method")
 
         if self.hub_address is not None and self.hub_zmq_port is not None:
@@ -549,7 +532,6 @@ class Interchange:
                         monitoring_radio.send(r['payload'])
                     elif r['type'] == 'heartbeat':
                         logger.debug("Manager %r sent heartbeat via results connection", manager_id)
-                        b_messages.append((p_message, r))
                     else:
                         logger.error("Interchange discarding result_queue message of unknown type: %s", r["type"])
 

parsl-2024.10.21/parsl/executors/high_throughput/manager_selector.py

@@ -0,0 +1,55 @@
+import random
+from abc import ABCMeta, abstractmethod
+from typing import Dict, List, Set
+
+from parsl.executors.high_throughput.manager_record import ManagerRecord
+
+
+class ManagerSelector(metaclass=ABCMeta):
+
+    @abstractmethod
+    def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
+        """ Sort a given list of managers.
+
+        Any operations pertaining to the sorting and rearrangement of the
+        interesting_managers Set should be performed here.
+        """
+        pass
+
+
+class RandomManagerSelector(ManagerSelector):
+
+    """Returns a shuffled list of interesting_managers
+
+    By default this strategy is used by the interchange. Works well
+    in distributing workloads equally across all availble compute
+    resources. The random workload strategy is not effective in
+    conjunction with elastic scaling behavior as the even task
+    distribution does not allow the scaling down of blocks, leading
+    to wasted resource consumption.
+    """
+
+    def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
+        c_manager_list = list(manager_list)
+        random.shuffle(c_manager_list)
+        return c_manager_list
+
+
+class BlockIdManagerSelector(ManagerSelector):
+
+    """Returns an interesting_managers list sorted by block ID
+
+    Observations:
+    1. BlockID manager selector helps with workloads that see a varying
+    amount of tasks over time. New blocks are prioritized with the
+    blockID manager selector, when used with 'htex_auto_scaling', results
+    in compute cost savings.
+
+    2. Doesn't really work with bag-of-tasks workloads. When all the tasks
+    are put into the queue upfront, all blocks operate at near full
+    utilization for the majority of the workload, which task goes where
+    doesn't really matter.
+    """
+
+    def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
+        return sorted(manager_list, key=lambda x: (ready_managers[x]['block_id'] is not None, ready_managers[x]['block_id']))

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/high_throughput/mpi_prefix_composer.py

@@ -1,5 +1,7 @@
 import logging
-from typing import Dict, List, Set, Tuple
+from typing import Dict, List, Tuple
+
+from parsl.executors.errors import InvalidResourceSpecification
 
 logger = logging.getLogger(__name__)
 
@@ -8,27 +10,6 @@ VALID_LAUNCHERS = ('srun',
                    'mpiexec')
 
 
-class MissingResourceSpecification(Exception):
-    """Exception raised when input is  not supplied a resource specification"""
-
-    def __init__(self, reason: str):
-        self.reason = reason
-
-    def __str__(self):
-        return f"Missing resource specification: {self.reason}"
-
-
-class InvalidResourceSpecification(Exception):
-    """Exception raised when Invalid input is supplied via resource specification"""
-
-    def __init__(self, invalid_keys: Set[str], message: str = ''):
-        self.invalid_keys = invalid_keys
-        self.message = message
-
-    def __str__(self):
-        return f"Invalid resource specification options supplied: {self.invalid_keys} {self.message}"
-
-
 def validate_resource_spec(resource_spec: Dict[str, str]):
     """Basic validation of keys in the resource_spec
 
@@ -40,7 +21,8 @@ def validate_resource_spec(resource_spec: Dict[str, str]):
     # empty resource_spec when mpi_mode is set causes parsl to hang
     # ref issue #3427
     if len(user_keys) == 0:
-        raise MissingResourceSpecification('MPI mode requires optional parsl_resource_specification keyword argument to be configured')
+        raise InvalidResourceSpecification(user_keys,
+                                           'MPI mode requires optional parsl_resource_specification keyword argument to be configured')
 
     legal_keys = set(("ranks_per_node",
                       "num_nodes",

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/high_throughput/process_worker_pool.py

@@ -362,7 +362,7 @@ class Manager:
                 if tasks == HEARTBEAT_CODE:
                     logger.debug("Got heartbeat from interchange")
                 elif tasks == DRAINED_CODE:
-                    logger.info("Got fulled drained message from interchange - setting kill flag")
+                    logger.info("Got fully drained message from interchange - setting kill flag")
                     kill_event.set()
                 else:
                     task_recv_counter += len(tasks)

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/threads.py

@@ -6,7 +6,7 @@ import typeguard
 
 from parsl.data_provider.staging import Staging
 from parsl.executors.base import ParslExecutor
-from parsl.executors.errors import UnsupportedFeatureError
+from parsl.executors.errors import InvalidResourceSpecification
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
@@ -54,7 +54,8 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
         if resource_specification:
             logger.error("Ignoring the resource specification. "
                          "Parsl resource specification is not supported in ThreadPool Executor.")
-            raise UnsupportedFeatureError('resource specification', 'ThreadPool Executor', None)
+            raise InvalidResourceSpecification(set(resource_specification.keys()),
+                                               "Parsl resource specification is not supported in ThreadPool Executor.")
 
         return self.executor.submit(func, *args, **kwargs)
 

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/executors/workqueue/executor.py

@@ -28,7 +28,7 @@ import parsl.utils as putils
 from parsl.data_provider.files import File
 from parsl.data_provider.staging import Staging
 from parsl.errors import OptionalModuleMissing
-from parsl.executors.errors import ExecutorError
+from parsl.executors.errors import ExecutorError, InvalidResourceSpecification
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.executors.workqueue import exec_parsl_function
 from parsl.process_loggers import wrap_with_logs
@@ -419,7 +419,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                 message = "Task resource specification only accepts these types of resources: {}".format(
                         ', '.join(acceptable_fields))
                 logger.error(message)
-                raise ExecutorError(self, message)
+                raise InvalidResourceSpecification(keys, message)
 
             # this checks that either all of the required resource types are specified, or
             # that none of them are: the `required_resource_types` are not actually required,
@@ -430,9 +430,10 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                 logger.error("Running with `autolabel=False`. In this mode, "
                              "task resource specification requires "
                              "three resources to be specified simultaneously: cores, memory, and disk")
-                raise ExecutorError(self, "Task resource specification requires "
-                                          "three resources to be specified simultaneously: cores, memory, and disk. "
-                                          "Try setting autolabel=True if you are unsure of the resource usage")
+                raise InvalidResourceSpecification(keys,
+                                                   "Task resource specification requires "
+                                                   "three resources to be specified simultaneously: cores, memory, and disk. "
+                                                   "Try setting autolabel=True if you are unsure of the resource usage")
 
             for k in keys:
                 if k == 'cores':

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/providers/kubernetes/kube.py

@@ -1,10 +1,5 @@
 import logging
-import time
-
-from parsl.providers.kubernetes.template import template_string
-
-logger = logging.getLogger(__name__)
-
+import uuid
 from typing import Any, Dict, List, Optional, Tuple
 
 import typeguard
@@ -12,7 +7,8 @@ import typeguard
 from parsl.errors import OptionalModuleMissing
 from parsl.jobs.states import JobState, JobStatus
 from parsl.providers.base import ExecutionProvider
-from parsl.utils import RepresentationMixin
+from parsl.providers.kubernetes.template import template_string
+from parsl.utils import RepresentationMixin, sanitize_dns_subdomain_rfc1123
 
 try:
     from kubernetes import client, config
@@ -20,6 +16,8 @@ try:
 except (ImportError, NameError, FileNotFoundError):
     _kubernetes_enabled = False
 
+logger = logging.getLogger(__name__)
+
 translate_table = {
     'Running': JobState.RUNNING,
     'Pending': JobState.PENDING,
@@ -161,7 +159,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         self.resources: Dict[object, Dict[str, Any]]
         self.resources = {}
 
-    def submit(self, cmd_string, tasks_per_node, job_name="parsl"):
+    def submit(self, cmd_string: str, tasks_per_node: int, job_name: str = "parsl.kube"):
         """ Submit a job
         Args:
              - cmd_string  :(String) - Name of the container to initiate
@@ -173,15 +171,19 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         Returns:
              - job_id: (string) Identifier for the job
         """
+        job_id = uuid.uuid4().hex[:8]
 
-        cur_timestamp = str(time.time() * 1000).split(".")[0]
-        job_name = "{0}-{1}".format(job_name, cur_timestamp)
-
-        if not self.pod_name:
-            pod_name = '{}'.format(job_name)
-        else:
-            pod_name = '{}-{}'.format(self.pod_name,
-                                      cur_timestamp)
+        pod_name = self.pod_name or job_name
+        try:
+            pod_name = sanitize_dns_subdomain_rfc1123(pod_name)
+        except ValueError:
+            logger.warning(
+                f"Invalid pod name '{pod_name}' for job '{job_id}', falling back to 'parsl.kube'"
+            )
+            pod_name = "parsl.kube"
+        pod_name = pod_name[:253 - 1 - len(job_id)]  # Leave room for the job ID
+        pod_name = pod_name.rstrip(".-")  # Remove trailing dot or hyphen after trim
+        pod_name = f"{pod_name}.{job_id}"
 
         formatted_cmd = template_string.format(command=cmd_string,
                                                worker_init=self.worker_init)
@@ -189,14 +191,14 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         logger.debug("Pod name: %s", pod_name)
         self._create_pod(image=self.image,
                          pod_name=pod_name,
-                         job_name=job_name,
+                         job_id=job_id,
                          cmd_string=formatted_cmd,
                          volumes=self.persistent_volumes,
                          service_account_name=self.service_account_name,
                          annotations=self.annotations)
-        self.resources[pod_name] = {'status': JobStatus(JobState.RUNNING)}
+        self.resources[job_id] = {'status': JobStatus(JobState.RUNNING), 'pod_name': pod_name}
 
-        return pod_name
+        return job_id
 
     def status(self, job_ids):
         """ Get the status of a list of jobs identified by the job identifiers
@@ -212,6 +214,9 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
             self._status()
         return [self.resources[jid]['status'] for jid in job_ids]
 
+    def _get_pod_name(self, job_id: str) -> str:
+        return self.resources[job_id]['pod_name']
+
     def cancel(self, job_ids):
         """ Cancels the jobs specified by a list of job ids
         Args:
@@ -221,7 +226,8 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         """
         for job in job_ids:
             logger.debug("Terminating job/pod: {0}".format(job))
-            self._delete_pod(job)
+            pod_name = self._get_pod_name(job)
+            self._delete_pod(pod_name)
 
             self.resources[job]['status'] = JobStatus(JobState.CANCELLED)
         rets = [True for i in job_ids]
@@ -242,7 +248,8 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         for jid in to_poll_job_ids:
             phase = None
             try:
-                pod = self.kube_client.read_namespaced_pod(name=jid, namespace=self.namespace)
+                pod_name = self._get_pod_name(jid)
+                pod = self.kube_client.read_namespaced_pod(name=pod_name, namespace=self.namespace)
             except Exception:
                 logger.exception("Failed to poll pod {} status, most likely because pod was terminated".format(jid))
                 if self.resources[jid]['status'] is JobStatus(JobState.RUNNING):
@@ -257,10 +264,10 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                 self.resources[jid]['status'] = JobStatus(status)
 
     def _create_pod(self,
-                    image,
-                    pod_name,
-                    job_name,
-                    port=80,
+                    image: str,
+                    pod_name: str,
+                    job_id: str,
+                    port: int = 80,
                     cmd_string=None,
                     volumes=[],
                     service_account_name=None,
@@ -269,7 +276,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         Args:
               - image (string) : Docker image to launch
               - pod_name (string) : Name of the pod
-              - job_name (string) : App label
+              - job_id (string) : Job ID
         KWargs:
              - port (integer) : Container port
         Returns:
@@ -299,7 +306,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                                                   )
         # Configure Pod template container
         container = client.V1Container(
-            name=pod_name,
+            name=job_id,
             image=image,
             resources=resources,
             ports=[client.V1ContainerPort(container_port=port)],
@@ -322,7 +329,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                                                    claim_name=volume[0])))
 
         metadata = client.V1ObjectMeta(name=pod_name,
-                                       labels={"app": job_name},
+                                       labels={"parsl-job-id": job_id},
                                        annotations=annotations)
         spec = client.V1PodSpec(containers=[container],
                                 image_pull_secrets=[secret],

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/tests/conftest.py

@@ -58,7 +58,7 @@ def tmpd_cwd_session(pytestconfig):
 
     config = re.sub(r"[^A-z0-9_-]+", "_", pytestconfig.getoption('config')[0])
     cwd = pathlib.Path(os.getcwd())
-    pytest_dir = cwd / ".pytest"
+    pytest_dir = cwd / "pytest-parsl"
     pytest_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
 
     test_dir_prefix = "parsltest-"

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,11 +1,9 @@
 import parsl
 from parsl.app.app import python_app
 from parsl.executors import WorkQueueExecutor
-from parsl.executors.errors import ExecutorError, UnsupportedFeatureError
+from parsl.executors.errors import InvalidResourceSpecification
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
-from parsl.executors.high_throughput.mpi_prefix_composer import (
-    InvalidResourceSpecification,
-)
+from parsl.executors.threads import ThreadPoolExecutor
 
 
 @python_app
@@ -27,11 +25,10 @@ def test_resource(n=2):
     try:
         fut.result()
     except InvalidResourceSpecification:
-        assert isinstance(executor, HighThroughputExecutor)
-    except UnsupportedFeatureError:
-        assert not isinstance(executor, WorkQueueExecutor)
-    except Exception as e:
-        assert isinstance(e, ExecutorError)
+        assert (
+            isinstance(executor, HighThroughputExecutor) or
+            isinstance(executor, WorkQueueExecutor) or
+            isinstance(executor, ThreadPoolExecutor))
 
     # Specify resources with wrong types
     # 'cpus' is incorrect, should be 'cores'
@@ -40,8 +37,7 @@ def test_resource(n=2):
     try:
         fut.result()
     except InvalidResourceSpecification:
-        assert isinstance(executor, HighThroughputExecutor)
-    except UnsupportedFeatureError:
-        assert not isinstance(executor, WorkQueueExecutor)
-    except Exception as e:
-        assert isinstance(e, ExecutorError)
+        assert (
+            isinstance(executor, HighThroughputExecutor) or
+            isinstance(executor, WorkQueueExecutor) or
+            isinstance(executor, ThreadPoolExecutor))

parsl-2024.10.21/parsl/tests/test_htex/test_block_manager_selector_unit.py

@@ -0,0 +1,20 @@
+import pytest
+
+from parsl.executors.high_throughput.manager_record import ManagerRecord
+from parsl.executors.high_throughput.manager_selector import BlockIdManagerSelector
+
+
+@pytest.mark.local
+def test_sort_managers():
+    ready_managers = {
+        b'manager1': {'block_id': 1},
+        b'manager2': {'block_id': None},
+        b'manager3': {'block_id': 3},
+        b'manager4': {'block_id': 2}
+    }
+
+    manager_list = {b'manager1', b'manager2', b'manager3', b'manager4'}
+    expected_sorted_list = [b'manager2', b'manager1', b'manager4', b'manager3']
+    manager_selector = BlockIdManagerSelector()
+    sorted_managers = manager_selector.sort_managers(ready_managers, manager_list)
+    assert sorted_managers == expected_sorted_list

{parsl-2024.10.7 → parsl-2024.10.21}/parsl/tests/test_htex/test_drain.py

@@ -13,7 +13,9 @@ from parsl.providers import LocalProvider
 # based around the expected drain period: the drain period
 # is TIME_CONST seconds, and the single executed task will
 # last twice that many number of seconds.
-TIME_CONST = 1
+TIME_CONST = 4
+
+CONNECTED_MANAGERS_POLL_MS = 100
 
 
 def local_config():
@@ -52,7 +54,7 @@ def test_drain(try_assert):
 
     # wait till we have a block running...
 
-    try_assert(lambda: len(htex.connected_managers()) == 1)
+    try_assert(lambda: len(htex.connected_managers()) == 1, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     managers = htex.connected_managers()
     assert managers[0]['active'], "The manager should be active"
@@ -63,7 +65,7 @@ def test_drain(try_assert):
     time.sleep(TIME_CONST)
 
     # this assert should happen *very fast* after the above delay...
-    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500)
+    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     # and the test task should still be running...
     assert not fut.done(), "The test task should still be running"
@@ -76,4 +78,4 @@ def test_drain(try_assert):
     # connected managers.
     # As with the above draining assert, this should happen very fast after
     # the task ends.
-    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500)
+    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)