parsl 2024.7.8__tar.gz → 2024.7.22__tar.gz

{parsl-2024.7.8/parsl.egg-info → parsl-2024.7.22}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.7.8
+Version: 2024.7.22
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.07.08.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.07.22.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.7.8 → parsl-2024.7.22}/parsl/dataflow/dflow.py

@@ -1277,6 +1277,23 @@ class DataFlowKernel:
             executor.shutdown()
             logger.info(f"Shut down executor {executor.label}")
 
+            if hasattr(executor, 'provider'):
+                if hasattr(executor.provider, 'script_dir'):
+                    logger.info(f"Closing channel(s) for {executor.label}")
+
+                    if hasattr(executor.provider, 'channels'):
+                        for channel in executor.provider.channels:
+                            logger.info(f"Closing channel {channel}")
+                            channel.close()
+                            logger.info(f"Closed channel {channel}")
+                    else:
+                        assert hasattr(executor.provider, 'channel'), "If provider has no .channels, it must have .channel"
+                        logger.info(f"Closing channel {executor.provider.channel}")
+                        executor.provider.channel.close()
+                        logger.info(f"Closed channel {executor.provider.channel}")
+
+                    logger.info(f"Closed executor channel(s) for {executor.label}")
+
         logger.info("Terminated executors")
         self.time_completed = datetime.datetime.now()
 

parsl-2024.7.22/parsl/executors/flux/flux_instance_manager.py

@@ -0,0 +1,56 @@
+"""Script meant to be the initial program of a Flux instance."""
+
+import argparse
+import logging
+import os
+from os.path import dirname
+from socket import gethostbyname, gethostname
+
+import zmq
+
+
+def main():
+    """Run a Flux instance to completion.
+
+    Send the path to the Flux Python package and the URI of the
+    encapsulating Flux instance.
+    """
+    # flux imports only available when launched under Flux instance
+    import flux
+    import flux.job
+
+    logging.basicConfig(
+        level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s"
+    )
+    parser = argparse.ArgumentParser()
+    parser.add_argument("protocol", help="Protocol of the parent executor's socket")
+    parser.add_argument("hostname", help="hostname of the parent executor's socket")
+    parser.add_argument("port", help="Port of the parent executor's socket")
+    args = parser.parse_args()
+    with zmq.Context() as context, context.socket(zmq.REQ) as socket:
+        socket.connect(
+            args.protocol + "://" + gethostbyname(args.hostname) + ":" + args.port
+        )
+        # send the path to the ``flux.job`` package
+        socket.send(dirname(dirname(os.path.realpath(flux.__file__))).encode())
+        logging.debug("Flux package path sent.")
+        # collect the encapsulating Flux instance's URI
+        local_uri = flux.Flux().attr_get("local-uri")
+        hostname = gethostname()
+        if args.hostname == hostname:
+            flux_uri = local_uri
+        else:
+            flux_uri = "ssh://" + gethostname() + local_uri.replace("local://", "")
+        logging.debug("Flux URI is %s", flux_uri)
+        response = socket.recv()  # get acknowledgment
+        logging.debug("Received acknowledgment %s", response)
+        socket.send(flux_uri.encode())  # send URI
+        logging.debug("URI sent. Blocking for response...")
+        response = socket.recv()  # wait for shutdown message
+        logging.debug("Response %s received, draining flux jobs...", response)
+        flux.Flux().rpc("job-manager.drain").get()
+        logging.debug("Flux jobs drained, exiting.")
+
+
+if __name__ == "__main__":
+    main()

{parsl-2024.7.8 → parsl-2024.7.22}/parsl/executors/high_throughput/executor.py

@@ -168,7 +168,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                  |      |         |             | batching      |    |         |
            Parsl<---Fut-|         |             | load-balancing|  result   exception
                      ^  |         |             | watchdogs     |    |         |
-                     |  |         |   Q_mngmnt  |               |    V         V
+                     |  |         |    Result   |               |    |         |
+                     |  |         |    Queue    |               |    V         V
                      |  |         |    Thread<--|-incoming_q<---|--- +---------+
                      |  |         |      |      |               |
                      |  |         |      |      |               |
@@ -429,20 +430,19 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
 
-        self._queue_management_thread = None
-        self._start_queue_management_thread()
+        self._result_queue_thread = None
+        self._start_result_queue_thread()
         self._start_local_interchange_process()
 
-        logger.debug("Created management thread: {}".format(self._queue_management_thread))
+        logger.debug("Created result queue thread: %s", self._result_queue_thread)
 
         self.initialize_scaling()
 
     @wrap_with_logs
-    def _queue_management_worker(self):
-        """Listen to the queue for task status messages and handle them.
+    def _result_queue_worker(self):
+        """Listen to the queue for task result messages and handle them.
 
-        Depending on the message, tasks will be updated with results, exceptions,
-        or updates. It expects the following messages:
+        Depending on the message, tasks will be updated with results or exceptions.
 
         .. code:: python
 
@@ -459,7 +459,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
         The `None` message is a die request.
         """
-        logger.debug("Queue management worker starting")
+        logger.debug("Result queue worker starting")
 
         while not self.bad_state_is_set:
             try:
@@ -528,7 +528,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                         else:
                             raise BadMessage("Message received with unknown type {}".format(msg['type']))
 
-        logger.info("Queue management worker finished")
+        logger.info("Result queue worker finished")
 
     def _start_local_interchange_process(self) -> None:
         """ Starts the interchange process locally
@@ -571,21 +571,21 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             raise Exception("Interchange failed to start")
         logger.debug("Got worker ports")
 
-    def _start_queue_management_thread(self):
-        """Method to start the management thread as a daemon.
+    def _start_result_queue_thread(self):
+        """Method to start the result queue thread as a daemon.
 
         Checks if a thread already exists, then starts it.
-        Could be used later as a restart if the management thread dies.
+        Could be used later as a restart if the result queue thread dies.
         """
-        if self._queue_management_thread is None:
-            logger.debug("Starting queue management thread")
-            self._queue_management_thread = threading.Thread(target=self._queue_management_worker, name="HTEX-Queue-Management-Thread")
-            self._queue_management_thread.daemon = True
-            self._queue_management_thread.start()
-            logger.debug("Started queue management thread")
+        if self._result_queue_thread is None:
+            logger.debug("Starting result queue thread")
+            self._result_queue_thread = threading.Thread(target=self._result_queue_worker, name="HTEX-Result-Queue-Thread")
+            self._result_queue_thread.daemon = True
+            self._result_queue_thread.start()
+            logger.debug("Started result queue thread")
 
         else:
-            logger.error("Management thread already exists, returning")
+            logger.error("Result queue thread already exists, returning")
 
     def hold_worker(self, worker_id: str) -> None:
         """Puts a worker on hold, preventing scheduling of additional tasks to it.
@@ -834,6 +834,23 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             logger.info("Unable to terminate Interchange process; sending SIGKILL")
             self.interchange_proc.kill()
 
+        logger.info("Closing ZMQ pipes")
+
+        # These pipes are used in a thread unsafe manner. If you have traced a
+        # problem to this block of code, you might consider what is happening
+        # with other threads that access these.
+
+        # incoming_q is not closed here because it is used by the results queue
+        # worker which is not shut down at this point.
+
+        if hasattr(self, 'outgoing_q'):
+            logger.info("Closing outgoing_q")
+            self.outgoing_q.close()
+
+        if hasattr(self, 'command_client'):
+            logger.info("Closing command client")
+            self.command_client.close()
+
         logger.info("Finished HighThroughputExecutor shutdown attempt")
 
     def get_usage_information(self):

{parsl-2024.7.8 → parsl-2024.7.22}/parsl/executors/high_throughput/process_worker_pool.py

@@ -9,6 +9,7 @@ import os
 import pickle
 import platform
 import queue
+import subprocess
 import sys
 import threading
 import time
@@ -733,7 +734,26 @@ def worker(
 
     # If desired, pin to accelerator
     if accelerator is not None:
-        os.environ["CUDA_VISIBLE_DEVICES"] = accelerator
+
+        # If CUDA devices, find total number of devices to allow for MPS
+        # See: https://developer.nvidia.com/system-management-interface
+        nvidia_smi_cmd = "nvidia-smi -L > /dev/null && nvidia-smi -L | wc -l"
+        nvidia_smi_ret = subprocess.run(nvidia_smi_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nvidia_smi_ret.returncode == 0:
+            num_cuda_devices = int(nvidia_smi_ret.stdout.split()[0])
+        else:
+            num_cuda_devices = None
+
+        try:
+            if num_cuda_devices is not None:
+                procs_per_cuda_device = pool_size // num_cuda_devices
+                partitioned_accelerator = str(int(accelerator) // procs_per_cuda_device)  # multiple workers will share a GPU
+                os.environ["CUDA_VISIBLE_DEVICES"] = partitioned_accelerator
+                logger.info(f'Pinned worker to partitioned cuda device: {partitioned_accelerator}')
+            else:
+                os.environ["CUDA_VISIBLE_DEVICES"] = accelerator
+        except (TypeError, ValueError, ZeroDivisionError):
+            os.environ["CUDA_VISIBLE_DEVICES"] = accelerator
         os.environ["ROCR_VISIBLE_DEVICES"] = accelerator
         os.environ["ZE_AFFINITY_MASK"] = accelerator
         os.environ["ZE_ENABLE_PCI_ID_DEVICE_ORDER"] = '1'

{parsl-2024.7.8 → parsl-2024.7.22}/parsl/executors/status_handling.py

@@ -59,20 +59,28 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             self.block_error_handler = block_error_handler
 
-        # errors can happen during the submit call to the provider; this is used
-        # to keep track of such errors so that they can be handled in one place
-        # together with errors reported by status()
-        self._simulated_status: Dict[str, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
 
         self._block_id_counter = AtomicIDCounter()
 
         self._tasks = {}  # type: Dict[object, Future]
+
+        self._last_poll_time = 0.0
+
+        # these four structures track, in loosely coordinated fashion, the
+        # existence of blocks and jobs and how to map between their
+        # identifiers.
         self.blocks_to_job_id = {}  # type: Dict[str, str]
         self.job_ids_to_block = {}  # type: Dict[str, str]
 
-        self._last_poll_time = 0.0
+        # errors can happen during the submit call to the provider; this is used
+        # to keep track of such errors so that they can be handled in one place
+        # together with errors reported by status()
+        self._simulated_status: Dict[str, JobStatus] = {}
+
+        # this stores an approximation (sometimes delayed) of the latest status
+        # of pending, active and recently terminated blocks
         self._status = {}  # type: Dict[str, JobStatus]
 
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
@@ -113,20 +121,6 @@ class BlockProviderExecutor(ParslExecutor):
         raise NotImplementedError("Classes inheriting from BlockProviderExecutor must implement "
                                   "outstanding()")
 
-    def status(self) -> Dict[str, JobStatus]:
-        """Return the status of all jobs/blocks currently known to this executor.
-
-        :return: a dictionary mapping block ids (in string) to job status
-        """
-        if self._provider:
-            block_ids, job_ids = self._get_block_and_job_ids()
-            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
-        else:
-            status = {}
-        status.update(self._simulated_status)
-
-        return status
-
     def set_bad_state_and_fail_all(self, exception: Exception):
         """Allows external error handlers to mark this executor as irrecoverably bad and cause
         all tasks submitted to it now and in the future to fail. The executor is responsible
@@ -180,7 +174,7 @@ class BlockProviderExecutor(ParslExecutor):
         # Filters first iterable by bool values in second
         return list(compress(to_kill, killed))
 
-    def scale_out(self, blocks: int = 1) -> List[str]:
+    def _scale_out(self, blocks: int = 1) -> List[str]:
         """Scales out the number of blocks by "blocks"
         """
         if not self.provider:
@@ -276,6 +270,20 @@ class BlockProviderExecutor(ParslExecutor):
             if delta_status:
                 self.send_monitoring_info(delta_status)
 
+    def status(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks currently known to this executor.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        if self._provider:
+            block_ids, job_ids = self._get_block_and_job_ids()
+            status = self._make_status_dict(block_ids, self._provider.status(job_ids))
+        else:
+            status = {}
+        status.update(self._simulated_status)
+
+        return status
+
     @property
     def status_facade(self) -> Dict[str, JobStatus]:
         """Return the status of all jobs/blocks of the executor of this poller.
@@ -304,7 +312,7 @@ class BlockProviderExecutor(ParslExecutor):
         return block_ids
 
     def scale_out_facade(self, n: int) -> List[str]:
-        block_ids = self.scale_out(n)
+        block_ids = self._scale_out(n)
         if block_ids is not None:
             new_status = {}
             for block_id in block_ids:

parsl-2024.7.22/parsl/tests/test_channels/test_dfk_close.py

@@ -0,0 +1,26 @@
+from unittest.mock import Mock
+
+import pytest
+
+import parsl
+from parsl.channels.base import Channel
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import LocalProvider
+
+
+@pytest.mark.local
+def test_dfk_close():
+
+    mock_channel = Mock(spec=Channel)
+
+    # block settings all 0 because the mock channel won't be able to
+    # do anything to make a block exist
+    p = LocalProvider(channel=mock_channel, init_blocks=0, min_blocks=0, max_blocks=0)
+
+    e = HighThroughputExecutor(provider=p)
+
+    c = parsl.Config(executors=[e])
+    with parsl.load(c):
+        pass
+
+    assert mock_channel.close.called

{parsl-2024.7.8 → parsl-2024.7.22}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.07.08'
+VERSION = '2024.07.22'

{parsl-2024.7.8 → parsl-2024.7.22/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.7.8
+Version: 2024.7.22
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.07.08.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.07.22.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.7.8 → parsl-2024.7.22}/parsl.egg-info/SOURCES.txt

@@ -319,6 +319,7 @@ parsl/tests/test_bash_apps/test_pipeline.py
 parsl/tests/test_bash_apps/test_std_uri.py
 parsl/tests/test_bash_apps/test_stdout.py
 parsl/tests/test_channels/__init__.py
+parsl/tests/test_channels/test_dfk_close.py
 parsl/tests/test_channels/test_large_output.py
 parsl/tests/test_checkpointing/__init__.py
 parsl/tests/test_checkpointing/test_periodic.py

parsl-2024.7.8/parsl/executors/flux/flux_instance_manager.py

@@ -1,57 +0,0 @@
-"""Script meant to be the initial program of a Flux instance."""
-
-import argparse
-import logging
-import os
-from os.path import dirname
-from socket import gethostbyname, gethostname
-
-import zmq
-
-
-def main():
-    """Run a Flux instance to completion.
-
-    Send the path to the Flux Python package and the URI of the
-    encapsulating Flux instance.
-    """
-    # flux imports only available when launched under Flux instance
-    import flux
-    import flux.job
-
-    logging.basicConfig(
-        level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s"
-    )
-    parser = argparse.ArgumentParser()
-    parser.add_argument("protocol", help="Protocol of the parent executor's socket")
-    parser.add_argument("hostname", help="hostname of the parent executor's socket")
-    parser.add_argument("port", help="Port of the parent executor's socket")
-    args = parser.parse_args()
-    context = zmq.Context()
-    socket = context.socket(zmq.REQ)
-    socket.connect(
-        args.protocol + "://" + gethostbyname(args.hostname) + ":" + args.port
-    )
-    # send the path to the ``flux.job`` package
-    socket.send(dirname(dirname(os.path.realpath(flux.__file__))).encode())
-    logging.debug("Flux package path sent.")
-    # collect the encapsulating Flux instance's URI
-    local_uri = flux.Flux().attr_get("local-uri")
-    hostname = gethostname()
-    if args.hostname == hostname:
-        flux_uri = local_uri
-    else:
-        flux_uri = "ssh://" + gethostname() + local_uri.replace("local://", "")
-    logging.debug("Flux URI is %s", flux_uri)
-    response = socket.recv()  # get acknowledgment
-    logging.debug("Received acknowledgment %s", response)
-    socket.send(flux_uri.encode())  # send URI
-    logging.debug("URI sent. Blocking for response...")
-    response = socket.recv()  # wait for shutdown message
-    logging.debug("Response %s received, draining flux jobs...", response)
-    flux.Flux().rpc("job-manager.drain").get()
-    logging.debug("Flux jobs drained, exiting.")
-
-
-if __name__ == "__main__":
-    main()