parsl 2024.7.15__py3-none-any.whl → 2024.7.29__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -113,14 +113,10 @@ class DataFlowKernel:
         self.monitoring: Optional[MonitoringHub]
         self.monitoring = config.monitoring
 
-        # hub address and port for interchange to connect
-        self.hub_address = None  # type: Optional[str]
-        self.hub_zmq_port = None  # type: Optional[int]
         if self.monitoring:
             if self.monitoring.logdir is None:
                 self.monitoring.logdir = self.run_dir
-            self.hub_address = self.monitoring.hub_address
-            self.hub_zmq_port = self.monitoring.start(self.run_id, self.run_dir, self.config.run_dir)
+            self.monitoring.start(self.run_id, self.run_dir, self.config.run_dir)
 
         self.time_began = datetime.datetime.now()
         self.time_completed: Optional[datetime.datetime] = None
@@ -1181,9 +1177,9 @@ class DataFlowKernel:
         for executor in executors:
             executor.run_id = self.run_id
             executor.run_dir = self.run_dir
-            executor.hub_address = self.hub_address
-            executor.hub_zmq_port = self.hub_zmq_port
             if self.monitoring:
+                executor.hub_address = self.monitoring.hub_address
+                executor.hub_zmq_port = self.monitoring.hub_zmq_port
                 executor.monitoring_radio = self.monitoring.radio
             if hasattr(executor, 'provider'):
                 if hasattr(executor.provider, 'script_dir'):
@@ -1460,8 +1456,6 @@ class DataFlowKernel:
         Returns:
              - dict containing, hashed -> future mappings
         """
-        self.memo_lookup_table = None
-
         if checkpointDirs:
             return self._load_checkpoints(checkpointDirs)
         else:

parsl/executors/base.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Optional
 
 from typing_extensions import Literal, Self
 
-from parsl.monitoring.radios import MonitoringRadio
+from parsl.monitoring.radios import MonitoringRadioSender
 
 
 class ParslExecutor(metaclass=ABCMeta):
@@ -52,7 +52,7 @@ class ParslExecutor(metaclass=ABCMeta):
         *,
         hub_address: Optional[str] = None,
         hub_zmq_port: Optional[int] = None,
-        monitoring_radio: Optional[MonitoringRadio] = None,
+        monitoring_radio: Optional[MonitoringRadioSender] = None,
         run_dir: str = ".",
         run_id: Optional[str] = None,
     ):
@@ -147,11 +147,11 @@ class ParslExecutor(metaclass=ABCMeta):
         self._hub_zmq_port = value
 
     @property
-    def monitoring_radio(self) -> Optional[MonitoringRadio]:
+    def monitoring_radio(self) -> Optional[MonitoringRadioSender]:
         """Local radio for sending monitoring messages
         """
         return self._monitoring_radio
 
     @monitoring_radio.setter
-    def monitoring_radio(self, value: Optional[MonitoringRadio]) -> None:
+    def monitoring_radio(self, value: Optional[MonitoringRadioSender]) -> None:
         self._monitoring_radio = value

parsl/executors/flux/executor.py

@@ -200,7 +200,6 @@ class FluxExecutor(ParslExecutor, RepresentationMixin):
                 raise EnvironmentError("Cannot find Flux installation in PATH")
         self.flux_path = os.path.abspath(flux_path)
         self._task_id_counter = itertools.count()
-        self._socket = zmq.Context().socket(zmq.REP)
         # Assumes a launch command cannot be None or empty
         self.launch_cmd = launch_cmd or self.DEFAULT_LAUNCH_CMD
         self._submission_queue: queue.Queue = queue.Queue()
@@ -213,7 +212,6 @@ class FluxExecutor(ParslExecutor, RepresentationMixin):
             args=(
                 self._submission_queue,
                 self._stop_event,
-                self._socket,
                 self.working_dir,
                 self.flux_executor_kwargs,
                 self.provider,
@@ -306,11 +304,13 @@ def _submit_wrapper(
 
     If an exception is thrown, error out all submitted tasks.
     """
-    try:
-        _submit_flux_jobs(submission_queue, stop_event, *args, **kwargs)
-    except Exception as exc:
-        _error_out_jobs(submission_queue, stop_event, exc)
-        raise
+    with zmq.Context() as ctx:
+        with ctx.socket(zmq.REP) as socket:
+            try:
+                _submit_flux_jobs(submission_queue, stop_event, socket, *args, **kwargs)
+            except Exception as exc:
+                _error_out_jobs(submission_queue, stop_event, exc)
+                raise
 
 
 def _error_out_jobs(

parsl/executors/high_throughput/executor.py

@@ -456,8 +456,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                "task_id" : <task_id>
                "exception" : serialized exception object, on failure
             }
-
-        The `None` message is a die request.
         """
         logger.debug("Result queue worker starting")
 
@@ -475,58 +473,53 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
             else:
 
-                if msgs is None:
-                    logger.debug("Got None, exiting")
-                    return
+                for serialized_msg in msgs:
+                    try:
+                        msg = pickle.loads(serialized_msg)
+                    except pickle.UnpicklingError:
+                        raise BadMessage("Message received could not be unpickled")
 
-                else:
-                    for serialized_msg in msgs:
+                    if msg['type'] == 'heartbeat':
+                        continue
+                    elif msg['type'] == 'result':
                         try:
-                            msg = pickle.loads(serialized_msg)
-                        except pickle.UnpicklingError:
-                            raise BadMessage("Message received could not be unpickled")
+                            tid = msg['task_id']
+                        except Exception:
+                            raise BadMessage("Message received does not contain 'task_id' field")
+
+                        if tid == -1 and 'exception' in msg:
+                            logger.warning("Executor shutting down due to exception from interchange")
+                            exception = deserialize(msg['exception'])
+                            self.set_bad_state_and_fail_all(exception)
+                            break
+
+                        task_fut = self.tasks.pop(tid)
+
+                        if 'result' in msg:
+                            result = deserialize(msg['result'])
+                            task_fut.set_result(result)
 
-                        if msg['type'] == 'heartbeat':
-                            continue
-                        elif msg['type'] == 'result':
+                        elif 'exception' in msg:
                             try:
-                                tid = msg['task_id']
-                            except Exception:
-                                raise BadMessage("Message received does not contain 'task_id' field")
-
-                            if tid == -1 and 'exception' in msg:
-                                logger.warning("Executor shutting down due to exception from interchange")
-                                exception = deserialize(msg['exception'])
-                                self.set_bad_state_and_fail_all(exception)
-                                break
-
-                            task_fut = self.tasks.pop(tid)
-
-                            if 'result' in msg:
-                                result = deserialize(msg['result'])
-                                task_fut.set_result(result)
-
-                            elif 'exception' in msg:
-                                try:
-                                    s = deserialize(msg['exception'])
-                                    # s should be a RemoteExceptionWrapper... so we can reraise it
-                                    if isinstance(s, RemoteExceptionWrapper):
-                                        try:
-                                            s.reraise()
-                                        except Exception as e:
-                                            task_fut.set_exception(e)
-                                    elif isinstance(s, Exception):
-                                        task_fut.set_exception(s)
-                                    else:
-                                        raise ValueError("Unknown exception-like type received: {}".format(type(s)))
-                                except Exception as e:
-                                    # TODO could be a proper wrapped exception?
-                                    task_fut.set_exception(
-                                        DeserializationError("Received exception, but handling also threw an exception: {}".format(e)))
-                            else:
-                                raise BadMessage("Message received is neither result or exception")
+                                s = deserialize(msg['exception'])
+                                # s should be a RemoteExceptionWrapper... so we can reraise it
+                                if isinstance(s, RemoteExceptionWrapper):
+                                    try:
+                                        s.reraise()
+                                    except Exception as e:
+                                        task_fut.set_exception(e)
+                                elif isinstance(s, Exception):
+                                    task_fut.set_exception(s)
+                                else:
+                                    raise ValueError("Unknown exception-like type received: {}".format(type(s)))
+                            except Exception as e:
+                                # TODO could be a proper wrapped exception?
+                                task_fut.set_exception(
+                                    DeserializationError("Received exception, but handling also threw an exception: {}".format(e)))
                         else:
-                            raise BadMessage("Message received with unknown type {}".format(msg['type']))
+                            raise BadMessage("Message received is neither result or exception")
+                    else:
+                        raise BadMessage("Message received with unknown type {}".format(msg['type']))
 
         logger.info("Result queue worker finished")
 

parsl/executors/high_throughput/interchange.py

@@ -410,6 +410,7 @@ class Interchange:
                 self._ready_managers[manager_id] = {'last_heartbeat': time.time(),
                                                     'idle_since': time.time(),
                                                     'block_id': None,
+                                                    'start_time': msg['start_time'],
                                                     'max_capacity': 0,
                                                     'worker_count': 0,
                                                     'active': True,

parsl/executors/high_throughput/manager_record.py

@@ -6,6 +6,7 @@ from typing_extensions import TypedDict
 
 class ManagerRecord(TypedDict, total=False):
     block_id: Optional[str]
+    start_time: float
     tasks: List[Any]
     worker_count: int
     max_capacity: int

parsl/executors/high_throughput/process_worker_pool.py

@@ -184,6 +184,7 @@ class Manager:
 
         self.uid = uid
         self.block_id = block_id
+        self.start_time = time.time()
 
         self.enable_mpi_mode = enable_mpi_mode
         self.mpi_launcher = mpi_launcher
@@ -263,6 +264,7 @@ class Manager:
                'worker_count': self.worker_count,
                'uid': self.uid,
                'block_id': self.block_id,
+               'start_time': self.start_time,
                'prefetch_capacity': self.prefetch_capacity,
                'max_capacity': self.worker_count + self.prefetch_capacity,
                'os': platform.system(),
@@ -732,17 +734,18 @@ def worker(
         os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
-    # If CUDA devices, find total number of devices to allow for MPS
-    # See: https://developer.nvidia.com/system-management-interface
-    nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
-    nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if nvidia_smi_ret.returncode == 0:
-        num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
-    else:
-        num_cuda_devices = None
-
     # If desired, pin to accelerator
     if accelerator is not None:
+
+        # If CUDA devices, find total number of devices to allow for MPS
+        # See: https://developer.nvidia.com/system-management-interface
+        nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
+        nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nvidia_smi_ret.returncode == 0:
+            num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
+        else:
+            num_cuda_devices = None
+
         try:
             if num_cuda_devices is not None:
                 procs_per_cuda_device = pool_size // num_cuda_devices

parsl/executors/status_handling.py

@@ -59,20 +59,28 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             self.block_error_handler = block_error_handler
 
-        # errors can happen during the submit call to the provider; this is used
-        # to keep track of such errors so that they can be handled in one place
-        # together with errors reported by status()
-        self._simulated_status: Dict[str, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
 
         self._block_id_counter = AtomicIDCounter()
 
         self._tasks = {}  # type: Dict[object, Future]
+
+        self._last_poll_time = 0.0
+
+        # these four structures track, in loosely coordinated fashion, the
+        # existence of blocks and jobs and how to map between their
+        # identifiers.
         self.blocks_to_job_id = {}  # type: Dict[str, str]
         self.job_ids_to_block = {}  # type: Dict[str, str]
 
-        self._last_poll_time = 0.0
+        # errors can happen during the submit call to the provider; this is used
+        # to keep track of such errors so that they can be handled in one place
+        # together with errors reported by status()
+        self._simulated_status: Dict[str, JobStatus] = {}
+
+        # this stores an approximation (sometimes delayed) of the latest status
+        # of pending, active and recently terminated blocks
         self._status = {}  # type: Dict[str, JobStatus]
 
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
@@ -113,20 +121,6 @@ class BlockProviderExecutor(ParslExecutor):
         raise NotImplementedError("Classes inheriting from BlockProviderExecutor must implement "
                                   "outstanding()")
 
-    def status(self) -> Dict[str, JobStatus]:
-        """Return the status of all jobs/blocks currently known to this executor.
-
-        :return: a dictionary mapping block ids (in string) to job status
-        """
-        if self._provider:
-            block_ids, job_ids = self._get_block_and_job_ids()
-            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
-        else:
-            status = {}
-        status.update(self._simulated_status)
-
-        return status
-
     def set_bad_state_and_fail_all(self, exception: Exception):
         """Allows external error handlers to mark this executor as irrecoverably bad and cause
         all tasks submitted to it now and in the future to fail. The executor is responsible
@@ -180,7 +174,17 @@ class BlockProviderExecutor(ParslExecutor):
         # Filters first iterable by bool values in second
         return list(compress(to_kill, killed))
 
-    def scale_out(self, blocks: int = 1) -> List[str]:
+    def scale_out_facade(self, n: int) -> List[str]:
+        block_ids = self._scale_out(n)
+        if block_ids is not None:
+            new_status = {}
+            for block_id in block_ids:
+                new_status[block_id] = JobStatus(JobState.PENDING)
+            self.send_monitoring_info(new_status)
+            self._status.update(new_status)
+        return block_ids
+
+    def _scale_out(self, blocks: int = 1) -> List[str]:
         """Scales out the number of blocks by "blocks"
         """
         if not self.provider:
@@ -199,15 +203,32 @@ class BlockProviderExecutor(ParslExecutor):
                 self._simulated_status[block_id] = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
         return block_ids
 
-    @abstractmethod
     def scale_in(self, blocks: int) -> List[str]:
         """Scale in method.
 
         Cause the executor to reduce the number of blocks by count.
 
+        The default implementation will kill blocks without regard to their
+        status or whether they are executing tasks. Executors with more
+        nuanced scaling strategies might overload this method to work with
+        that strategy - see the HighThroughputExecutor for an example of that.
+
         :return: A list of block ids corresponding to the blocks that were removed.
         """
-        pass
+        # Obtain list of blocks to kill
+        to_kill = list(self.blocks_to_job_id.keys())[:blocks]
+        kill_ids = [self.blocks_to_job_id[block] for block in to_kill]
+
+        # Cancel the blocks provisioned
+        if self.provider:
+            logger.info(f"Scaling in jobs: {kill_ids}")
+            r = self.provider.cancel(kill_ids)
+            job_ids = self._filter_scale_in_ids(kill_ids, r)
+            block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
+            return block_ids_killed
+        else:
+            logger.error("No execution provider available to scale in")
+            return []
 
     def _launch_block(self, block_id: str) -> Any:
         launch_cmd = self._get_launch_command(block_id)
@@ -243,7 +264,7 @@ class BlockProviderExecutor(ParslExecutor):
         # Send monitoring info for HTEX when monitoring enabled
         if self.monitoring_radio:
             msg = self.create_monitoring_info(status)
-            logger.debug("Sending message {} to hub from job status poller".format(msg))
+            logger.debug("Sending block monitoring message: %r", msg)
             self.monitoring_radio.send((MessageType.BLOCK_INFO, msg))
 
     def create_monitoring_info(self, status: Dict[str, JobStatus]) -> Sequence[object]:
@@ -276,6 +297,20 @@ class BlockProviderExecutor(ParslExecutor):
             if delta_status:
                 self.send_monitoring_info(delta_status)
 
+    def status(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks currently known to this executor.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        if self._provider:
+            block_ids, job_ids = self._get_block_and_job_ids()
+            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
+        else:
+            status = {}
+        status.update(self._simulated_status)
+
+        return status
+
     @property
     def status_facade(self) -> Dict[str, JobStatus]:
         """Return the status of all jobs/blocks of the executor of this poller.
@@ -302,13 +337,3 @@ class BlockProviderExecutor(ParslExecutor):
                 del self._status[block_id]
             self.send_monitoring_info(new_status)
         return block_ids
-
-    def scale_out_facade(self, n: int) -> List[str]:
-        block_ids = self.scale_out(n)
-        if block_ids is not None:
-            new_status = {}
-            for block_id in block_ids:
-                new_status[block_id] = JobStatus(JobState.PENDING)
-            self.send_monitoring_info(new_status)
-            self._status.update(new_status)
-        return block_ids

parsl/executors/taskvine/executor.py

@@ -573,24 +573,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return 1
 
-    def scale_in(self, count: int) -> List[str]:
-        """Scale in method. Cancel a given number of blocks
-        """
-        # Obtain list of blocks to kill
-        to_kill = list(self.blocks_to_job_id.keys())[:count]
-        kill_ids = [self.blocks_to_job_id[block] for block in to_kill]
-
-        # Cancel the blocks provisioned
-        if self.provider:
-            logger.info(f"Scaling in jobs: {kill_ids}")
-            r = self.provider.cancel(kill_ids)
-            job_ids = self._filter_scale_in_ids(kill_ids, r)
-            block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
-            return block_ids_killed
-        else:
-            logger.error("No execution provider available to scale")
-            return []
-
     def shutdown(self, *args, **kwargs):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the TaskVine system submission.

parsl/executors/workqueue/executor.py

@@ -689,24 +689,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return self.scaling_cores_per_worker
 
-    def scale_in(self, count: int) -> List[str]:
-        """Scale in method.
-        """
-        # Obtain list of blocks to kill
-        to_kill = list(self.blocks_to_job_id.keys())[:count]
-        kill_ids = [self.blocks_to_job_id[block] for block in to_kill]
-
-        # Cancel the blocks provisioned
-        if self.provider:
-            logger.info(f"Scaling in jobs: {kill_ids}")
-            r = self.provider.cancel(kill_ids)
-            job_ids = self._filter_scale_in_ids(kill_ids, r)
-            block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
-            return block_ids_killed
-        else:
-            logger.error("No execution provider available to scale in")
-            return []
-
     def shutdown(self, *args, **kwargs):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the Work Queue system submission.

parsl/monitoring/monitoring.py

@@ -13,7 +13,7 @@ import typeguard
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.radios import MultiprocessingQueueRadio
+from parsl.monitoring.radios import MultiprocessingQueueRadioSender
 from parsl.monitoring.router import router_starter
 from parsl.monitoring.types import AddressedMonitoringMessage
 from parsl.multiprocessing import ForkProcess, SizedQueue
@@ -105,7 +105,7 @@ class MonitoringHub(RepresentationMixin):
         self.resource_monitoring_enabled = resource_monitoring_enabled
         self.resource_monitoring_interval = resource_monitoring_interval
 
-    def start(self, run_id: str, dfk_run_dir: str, config_run_dir: Union[str, os.PathLike]) -> int:
+    def start(self, run_id: str, dfk_run_dir: str, config_run_dir: Union[str, os.PathLike]) -> None:
 
         logger.debug("Starting MonitoringHub")
 
@@ -187,7 +187,7 @@ class MonitoringHub(RepresentationMixin):
         self.filesystem_proc.start()
         logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
 
-        self.radio = MultiprocessingQueueRadio(self.block_msgs)
+        self.radio = MultiprocessingQueueRadioSender(self.block_msgs)
 
         try:
             comm_q_result = comm_q.get(block=True, timeout=120)
@@ -207,7 +207,7 @@ class MonitoringHub(RepresentationMixin):
 
         logger.info("Monitoring Hub initialized")
 
-        return zmq_port
+        self.hub_zmq_port = zmq_port
 
     # TODO: tighten the Any message format
     def send(self, mtype: MessageType, message: Any) -> None:

parsl/monitoring/radios.py

@@ -15,14 +15,14 @@ _db_manager_excepts: Optional[Exception]
 logger = logging.getLogger(__name__)
 
 
-class MonitoringRadio(metaclass=ABCMeta):
+class MonitoringRadioSender(metaclass=ABCMeta):
     @abstractmethod
     def send(self, message: object) -> None:
         pass
 
 
-class FilesystemRadio(MonitoringRadio):
-    """A MonitoringRadio that sends messages over a shared filesystem.
+class FilesystemRadioSender(MonitoringRadioSender):
+    """A MonitoringRadioSender that sends messages over a shared filesystem.
 
     The messsage directory structure is based on maildir,
     https://en.wikipedia.org/wiki/Maildir
@@ -36,7 +36,7 @@ class FilesystemRadio(MonitoringRadio):
     This avoids a race condition of reading partially written messages.
 
     This radio is likely to give higher shared filesystem load compared to
-    the UDPRadio, but should be much more reliable.
+    the UDP radio, but should be much more reliable.
     """
 
     def __init__(self, *, monitoring_url: str, source_id: int, timeout: int = 10, run_dir: str):
@@ -66,7 +66,7 @@ class FilesystemRadio(MonitoringRadio):
         os.rename(tmp_filename, new_filename)
 
 
-class HTEXRadio(MonitoringRadio):
+class HTEXRadioSender(MonitoringRadioSender):
 
     def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
         """
@@ -120,7 +120,7 @@ class HTEXRadio(MonitoringRadio):
         return
 
 
-class UDPRadio(MonitoringRadio):
+class UDPRadioSender(MonitoringRadioSender):
 
     def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
         """
@@ -174,7 +174,7 @@ class UDPRadio(MonitoringRadio):
         return
 
 
-class MultiprocessingQueueRadio(MonitoringRadio):
+class MultiprocessingQueueRadioSender(MonitoringRadioSender):
     """A monitoring radio which connects over a multiprocessing Queue.
     This radio is intended to be used on the submit side, where components
     in the submit process, or processes launched by multiprocessing, will have

parsl/monitoring/remote.py

@@ -8,10 +8,10 @@ from typing import Any, Callable, Dict, List, Sequence, Tuple
 
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.radios import (
-    FilesystemRadio,
-    HTEXRadio,
-    MonitoringRadio,
-    UDPRadio,
+    FilesystemRadioSender,
+    HTEXRadioSender,
+    MonitoringRadioSender,
+    UDPRadioSender,
 )
 from parsl.multiprocessing import ForkProcess
 from parsl.process_loggers import wrap_with_logs
@@ -100,17 +100,17 @@ def monitor_wrapper(*,
     return (wrapped, args, new_kwargs)
 
 
-def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadio:
-    radio: MonitoringRadio
+def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadioSender:
+    radio: MonitoringRadioSender
     if radio_mode == "udp":
-        radio = UDPRadio(monitoring_hub_url,
-                         source_id=task_id)
+        radio = UDPRadioSender(monitoring_hub_url,
+                               source_id=task_id)
     elif radio_mode == "htex":
-        radio = HTEXRadio(monitoring_hub_url,
-                          source_id=task_id)
+        radio = HTEXRadioSender(monitoring_hub_url,
+                                source_id=task_id)
     elif radio_mode == "filesystem":
-        radio = FilesystemRadio(monitoring_url=monitoring_hub_url,
-                                source_id=task_id, run_dir=run_dir)
+        radio = FilesystemRadioSender(monitoring_url=monitoring_hub_url,
+                                      source_id=task_id, run_dir=run_dir)
     else:
         raise RuntimeError(f"Unknown radio mode: {radio_mode}")
     return radio

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -44,8 +44,8 @@ def test_row_counts():
     #   the latter is what i'm most suspicious of in my present investigation
 
     # dig out the interchange port...
-    hub_address = parsl.dfk().hub_address
-    hub_zmq_port = parsl.dfk().hub_zmq_port
+    hub_address = parsl.dfk().monitoring.hub_address
+    hub_zmq_port = parsl.dfk().monitoring.hub_zmq_port
 
     # this will send a string to a new socket connection
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.07.15'
+VERSION = '2024.07.29'

{parsl-2024.7.15.data → parsl-2024.7.29.data}/scripts/interchange.py

@@ -410,6 +410,7 @@ class Interchange:
                 self._ready_managers[manager_id] = {'last_heartbeat': time.time(),
                                                     'idle_since': time.time(),
                                                     'block_id': None,
+                                                    'start_time': msg['start_time'],
                                                     'max_capacity': 0,
                                                     'worker_count': 0,
                                                     'active': True,

{parsl-2024.7.15.data → parsl-2024.7.29.data}/scripts/process_worker_pool.py

@@ -184,6 +184,7 @@ class Manager:
 
         self.uid = uid
         self.block_id = block_id
+        self.start_time = time.time()
 
         self.enable_mpi_mode = enable_mpi_mode
         self.mpi_launcher = mpi_launcher
@@ -263,6 +264,7 @@ class Manager:
                'worker_count': self.worker_count,
                'uid': self.uid,
                'block_id': self.block_id,
+               'start_time': self.start_time,
                'prefetch_capacity': self.prefetch_capacity,
                'max_capacity': self.worker_count + self.prefetch_capacity,
                'os': platform.system(),
@@ -732,17 +734,18 @@ def worker(
         os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
-    # If CUDA devices, find total number of devices to allow for MPS
-    # See: https://developer.nvidia.com/system-management-interface
-    nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
-    nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if nvidia_smi_ret.returncode == 0:
-        num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
-    else:
-        num_cuda_devices = None
-
     # If desired, pin to accelerator
     if accelerator is not None:
+
+        # If CUDA devices, find total number of devices to allow for MPS
+        # See: https://developer.nvidia.com/system-management-interface
+        nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
+        nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nvidia_smi_ret.returncode == 0:
+            num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
+        else:
+            num_cuda_devices = None
+
         try:
             if num_cuda_devices is not None:
                 procs_per_cuda_device = pool_size // num_cuda_devices

{parsl-2024.7.15.dist-info → parsl-2024.7.29.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.7.15
+Version: 2024.7.29
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.07.15.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.07.29.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.7.15.dist-info → parsl-2024.7.29.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=MyaEcEq-Qf860u7V98u-PZrPNdtzOZL_NW6EhIJnmfQ,1937
 parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=91FjQiTUY383ueAjkBAgE21My9nba6SP2a2SrbB1r1Q,11250
-parsl/version.py,sha256=xBBY22CXKXmBYJqrmCPAgPlHvalhorEzfXaNGRSVeQU,131
+parsl/version.py,sha256=1_lxZuThXb8Z-pTgQtbYaRM-j9czEj-Gcgyrj-lOHRA,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
 parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
@@ -62,7 +62,7 @@ parsl/data_provider/staging.py,sha256=ZDZuuFg38pjUStegKPcvPsfGp3iMeReMzfU6DSwtJj
 parsl/data_provider/zip.py,sha256=S4kVuH9lxAegRURYbvIUR7EYYBOccyslaqyCrVWUBhw,4497
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/dataflow/dependency_resolvers.py,sha256=Om8Dgh7a0ZwgXAc6TlhxLSzvxXHDlNNV1aBNiD3JTNY,3325
-parsl/dataflow/dflow.py,sha256=jgNOIk3xXz90RXwC38ujMz7092XRdLFv5BrMyALYhps,68513
+parsl/dataflow/dflow.py,sha256=rdDWhhsPhCkCqxMIhagBKOZMQKDXsDxp7hnTgY_LPqI,68256
 parsl/dataflow/errors.py,sha256=9SxVhIJY_53FQx8x4OU8UA8nd7lvUbDllH7KfMXpYaY,2177
 parsl/dataflow/futures.py,sha256=08LuP-HFiHBIZmeKCjlsazw_WpQ5fwevrU2_WbidkYw,6080
 parsl/dataflow/memoization.py,sha256=l9uw1Bu50GucBF70M5relpGKFkE4dIM9T3R1KrxW0v0,9583
@@ -70,25 +70,25 @@ parsl/dataflow/rundirs.py,sha256=7aUg1cb0LLTocQxOdBzwtn7a8bIgpdMD5rjZV55UwaQ,115
 parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
 parsl/dataflow/taskrecord.py,sha256=-FuujdZQ1y5GSc-PJ91QKGT-Kp0lrg70MFDoxpbWI1Q,3113
 parsl/executors/__init__.py,sha256=Cg8e-F2NUaBD8A9crDAXKCSdoBEwQVIdgm4FlXd-wvk,476
-parsl/executors/base.py,sha256=10xMzqVa2vV7muet08Tm1iHBZ4m2jCESPPiRnbwzGUk,5120
+parsl/executors/base.py,sha256=BECTvBfVRDATyhBmMdcH75xoFhx-LO3rQYawVhaUJ6M,5144
 parsl/executors/errors.py,sha256=xVswxgi7vmJcUMCeYDAPK8sQT2kHFFROVoOr0dnmcWE,2098
-parsl/executors/status_handling.py,sha256=o3rBvCd_j87ZkUD9SZlaHjwg-WMPRzg3gkMETt-HNis,13473
+parsl/executors/status_handling.py,sha256=BPv51pJbscV-HdOWgRN5JRPPyOm1b4m3qBbz7pTQjpc,14662
 parsl/executors/threads.py,sha256=hJt1LzxphqX4fe_9R9Cf1MU0lepWTU_eJe8O665B0Xo,3352
 parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
 parsl/executors/flux/execute_parsl_task.py,sha256=gRN7F4HhdrKQ-bvn4wXrquBzFOp_9WF88hMIeUaRg5I,1553
-parsl/executors/flux/executor.py,sha256=gPq49CQwtSZYZggLZ0dCXdpUlllKHJbvR8WRKeGh9xE,16977
+parsl/executors/flux/executor.py,sha256=8_xakLUu5zNJAHL0LbeTCFEWqWzRK1eE-3ep4GIIIrY,17017
 parsl/executors/flux/flux_instance_manager.py,sha256=5T3Rp7ZM-mlT0Pf0Gxgs5_YmnaPrSF9ec7zvRfLfYJw,2129
 parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/high_throughput/errors.py,sha256=Sak8e8UpiEcXefUjMHbhyXc4Rn7kJtOoh7L8wreBQdk,1638
-parsl/executors/high_throughput/executor.py,sha256=bCtw_p2f1ztnqQiChKJBOiPyc6aKK39yRXSp5uFpRzk,38185
-parsl/executors/high_throughput/interchange.py,sha256=IRuiaBmks_R4cU-Sx0Q_Fjv4PdFtzU05GiPdeJstOoA,30578
-parsl/executors/high_throughput/manager_record.py,sha256=9XppKjDW0DJ7SMkPNxsiDs-HvXGPLrTg6Ceyh4b6gNs,433
+parsl/executors/high_throughput/executor.py,sha256=fY-OuzStEgyM-ao11debyhbB3pRk4frEmXdcmBHWsvQ,37834
+parsl/executors/high_throughput/interchange.py,sha256=vmsUEAgqshi_K5n9oqb-bOGIt9BOFW32Ei_3Ur1C8EE,30663
+parsl/executors/high_throughput/manager_record.py,sha256=yn3L8TUJFkgm2lX1x0SeS9mkvJowC0s2VIMCFiU7ThM,455
 parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
 parsl/executors/high_throughput/mpi_executor.py,sha256=V07t1GOzFhcwdlZGuYUPqc1NarSr-vUbsNzbK4Cj0m8,3882
 parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=hah_IznfFqk-rzuHWmg6aiF_saiDRrpW-aSo4kH9Nso,4854
 parsl/executors/high_throughput/mpi_resource_management.py,sha256=LFBbJ3BnzTcY_v-jNu30uoIB2Enk4cleN4ygY3dncjY,8194
 parsl/executors/high_throughput/probe.py,sha256=TNpGTXb4_DEeg_h-LHu4zEKi1-hffboxvKcZUl2OZGk,2751
-parsl/executors/high_throughput/process_worker_pool.py,sha256=P1ZqQOyEpfvXxtfsevGpJvPH_PIxso3Mh0u8PyRbwD8,42958
+parsl/executors/high_throughput/process_worker_pool.py,sha256=3s-Ouo3ZEhod7hon8euyL37t1DbP5pSVjXyC23DSN_0,43075
 parsl/executors/high_throughput/zmq_pipes.py,sha256=tAjQB3aNVMuTXziN3dbJWre46YpXgliD55qMBbhYTLU,8581
 parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
 parsl/executors/radical/executor.py,sha256=426cMt6d8uJFZ_7Ub1kCslaND4OKtBX5WZdz-0RXjMk,22554
@@ -97,7 +97,7 @@ parsl/executors/radical/rpex_worker.py,sha256=qli6i6ejKubTSv3lAE3YiW8RlkHrfl4Jhr
 parsl/executors/taskvine/__init__.py,sha256=9rwp3M8B0YyEhZMLO0RHaNw7u1nc01WHbXLqnBTanu0,293
 parsl/executors/taskvine/errors.py,sha256=euIYkSslrNSI85kyi2s0xzOaO9ik4c1fYHstMIeiBJk,652
 parsl/executors/taskvine/exec_parsl_function.py,sha256=ftGdJU78lKPPkphSHlEi4rj164mhuMHJjghVqfgeXKk,7085
-parsl/executors/taskvine/executor.py,sha256=lZXfXXzkufLsoxAhf3hOMx05qTmwgd-osh2TMy1S4MM,31722
+parsl/executors/taskvine/executor.py,sha256=XsUzFWgFWbxf06jDLMAKiXUF-F1_VLheZ5jhAwSphZk,30977
 parsl/executors/taskvine/factory.py,sha256=rWpEoFphLzqO3HEYyDEbQa14iyvgkdZg7hLZuaY39gQ,2638
 parsl/executors/taskvine/factory_config.py,sha256=AbE2fN2snrF5ITYrrS4DnGn2XkJHUFr_17DYHDHIwq0,3693
 parsl/executors/taskvine/manager.py,sha256=fwRSgYWpbsnr5jXlzvX0sQjOqryqn_77K_svJJ1HJ2U,25631
@@ -106,7 +106,7 @@ parsl/executors/taskvine/utils.py,sha256=iSrIogeiauL3UNy_9tiZp1cBSNn6fIJkMYQRVi1
 parsl/executors/workqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/workqueue/errors.py,sha256=XO2naYhAsHHyiOBH6hpObg3mPNDmvMoFqErsj0-v7jc,541
 parsl/executors/workqueue/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
-parsl/executors/workqueue/executor.py,sha256=K5q-poZU37LVF4YhX34jKVMIBw5jpBfQ1rUadgPeQBU,50519
+parsl/executors/workqueue/executor.py,sha256=YaY_U5DxXU2NbswmlrcJ2BtXvSbV0ElM9ZlQzP_F_BU,49803
 parsl/executors/workqueue/parsl_coprocess.py,sha256=cF1UmTgVLoey6QzBcbYgEiEsRidSaFfuO54f1HFw_EM,5737
 parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1Mn1trSP41rtJguXxPGfHI,735
 parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -122,9 +122,9 @@ parsl/launchers/launchers.py,sha256=VB--fiVv_IQne3DydTMSdGUY0o0g69puAs-Hd3mJ2vo,
 parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
 parsl/monitoring/db_manager.py,sha256=qPdW_MINycSn6MxxFk2s_R-t8g1cbJhxncVR5mDgeGs,37011
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
-parsl/monitoring/monitoring.py,sha256=TCJsDbD3bLU77QrCqfCkLfC0e3Ih0FQLvtbYtgDs3RE,13522
-parsl/monitoring/radios.py,sha256=LkSW8LAW6MT9mNdHgRpUDPKkLcO-nyGU6uzWv3TLloE,5847
-parsl/monitoring/remote.py,sha256=IWDYCV8du2yB-YTqXjqGWELfi07ZWSENGq8mc4c4it0,13699
+parsl/monitoring/monitoring.py,sha256=ggffHhtgh96lSmryvjjjaHg7oV54Ci-kz4wWOPJviKU,13548
+parsl/monitoring/radios.py,sha256=VaVZEPDkLsJ3Jp2CFo5kQ94mxiqpN9xltKl8lnKksiQ,5908
+parsl/monitoring/remote.py,sha256=qH1N3My8F473CHKxFrA2CsaL0Uege26tQi9-KrWQrr4,13771
 parsl/monitoring/router.py,sha256=l1LBT1hxCWQ2mxCR-PtxwYJ905In61E4pPJB2K2J7kM,9554
 parsl/monitoring/types.py,sha256=_WGizCTgQVOkJ2dvNfsvHpYBj21Ky3bJsmyIskIx10I,631
 parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -355,7 +355,7 @@ parsl/tests/test_monitoring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 parsl/tests/test_monitoring/test_app_names.py,sha256=ayyxySGWpKSe9dDw2UeJo1dicxjpALRuLsJfprZV4Eg,2174
 parsl/tests/test_monitoring/test_basic.py,sha256=lGyHEJt_rokawv_XeAx-bxV84IlZUFR4KI0PQAiLsFg,3714
 parsl/tests/test_monitoring/test_db_locks.py,sha256=3s3c1xhKo230ZZIJ3f1Ca4U7LcEdXnanOGVXQyNlk2U,2895
-parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=E7T2yEyA_Ti8WxDrGZKya-8m8Hfe9HobVBsCe2BFbKM,3414
+parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=--3-pQUvXXbkr8v_BEJoPvVvNly1oXvrD2nJh6yl_0M,3436
 parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=_WjymTgxWvZZwQpJQ3L2gmEt5VUkTss0hOT153AssdQ,2746
 parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=ZnO1sFSwlWUBHX64C_zwfTVRVC_UFNlU4h0POgx6NEo,2005
 parsl/tests/test_monitoring/test_memoization_representation.py,sha256=dknv2nO7pNZ1jGxWGsC_AW3rs90gjMIeC5d7pIJ75Xc,2645
@@ -467,13 +467,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
 parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
 parsl/usage_tracking/usage.py,sha256=qNEJ7nPimqd3Y7OWFLdYmNwJ6XDKlyfV_fTzasxsQw8,8690
-parsl-2024.7.15.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
-parsl-2024.7.15.data/scripts/interchange.py,sha256=n0aOHLX64DEWx-OA4vWrYRVZfmaz8Rc8haNtafbgh4k,30565
-parsl-2024.7.15.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
-parsl-2024.7.15.data/scripts/process_worker_pool.py,sha256=pfIQ_JzqjviaiTfVI49qw4qy8FBS8AavN_12oL8DyzE,42944
-parsl-2024.7.15.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.7.15.dist-info/METADATA,sha256=bagqkFFK8EeAICbm5afqQ4--DJWNZ_900VszWxbxsZk,4124
-parsl-2024.7.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.7.15.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.7.15.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.7.15.dist-info/RECORD,,
+parsl-2024.7.29.data/scripts/exec_parsl_function.py,sha256=RUkJ4JSJAjr7YyRZ58zhMdg8cR5dVV9odUl3AuzNf3k,7802
+parsl-2024.7.29.data/scripts/interchange.py,sha256=q3V1mqr0BC_CzsNfebTKFD5tyE0birXUvZh-bk05vLQ,30650
+parsl-2024.7.29.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
+parsl-2024.7.29.data/scripts/process_worker_pool.py,sha256=78QKnV5KbY_vcteC6k60gpDE4wEk6hsciet_qzs9QoU,43061
+parsl-2024.7.29.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.7.29.dist-info/METADATA,sha256=DN5bT4CBKxmmsnscGT_y61gVnmzLEGRZDg95HFbxD54,4124
+parsl-2024.7.29.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.7.29.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.7.29.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.7.29.dist-info/RECORD,,