parsl 2024.4.1__py3-none-any.whl → 2024.4.15__py3-none-any.whl

parsl/data_provider/data_manager.py

@@ -7,6 +7,7 @@ from parsl.data_provider.files import File
 from parsl.data_provider.file_noop import NoOpFileStaging
 from parsl.data_provider.ftp import FTPSeparateTaskStaging
 from parsl.data_provider.http import HTTPSeparateTaskStaging
+from parsl.data_provider.zip import ZipFileStaging
 from parsl.data_provider.staging import Staging
 
 if TYPE_CHECKING:
@@ -17,7 +18,7 @@ logger = logging.getLogger(__name__)
 # these will be shared between all executors that do not explicitly
 # override, so should not contain executor-specific state
 default_staging: List[Staging]
-default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging()]
+default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging(), ZipFileStaging()]
 
 
 class DataManager:

parsl/data_provider/zip.py

@@ -0,0 +1,104 @@
+import filelock
+import logging
+import os
+import parsl
+import zipfile
+
+from typing import Tuple
+
+from parsl.data_provider.staging import Staging
+from parsl.data_provider.files import File
+from parsl.errors import ParslError
+
+
+logger = logging.getLogger(__name__)
+
+
+class ZipAuthorityError(ParslError):
+    def __init__(self, file):
+        self.file = file
+
+    def __str__(self):
+        return f"ZipFileStaging cannot stage Files with an authority (netloc) section ({self.file.netloc}), for {self.file.url}"
+
+
+class ZipFileStaging(Staging):
+    """A stage-out provider for zip files.
+
+    This provider will stage out files by writing them into the specified zip
+    file.
+
+    The filename of both the zip file and the file contained in that zip are
+    specified using a zip: URL, like this:
+
+    zip:/tmp/foo/this.zip/inside/here.txt
+
+    This URL names a zip file ``/tmp/foo/this.zip`` containing a file
+    ``inside/here.txt``.
+
+    The provider will use the Python filelock package to lock the zip file so
+    that it does not conflict with other instances of itself. This lock will
+    not protect against other modifications to the zip file.
+    """
+
+    def can_stage_out(self, file: File) -> bool:
+        logger.debug("archive provider checking File {}".format(repr(file)))
+
+        # First check if this is the scheme we care about
+        if file.scheme != "zip":
+            return False
+
+        # This is some basic validation to check that the user isn't specifying
+        # an authority section and expecting it to mean something.
+        if file.netloc != "":
+            raise ZipAuthorityError(file)
+
+        # If we got this far, we can stage this file
+        return True
+
+    def stage_out(self, dm, executor, file, parent_fut):
+        assert file.scheme == 'zip'
+
+        zip_path, inside_path = zip_path_split(file.path)
+
+        working_dir = dm.dfk.executors[executor].working_dir
+
+        if working_dir:
+            file.local_path = os.path.join(working_dir, inside_path)
+
+            # TODO: I think its the right behaviour that a staging out provider should create the directory structure
+            # for the file to be placed in?
+            os.makedirs(os.path.dirname(file.local_path), exist_ok=True)
+        else:
+            raise RuntimeError("zip file staging requires a working_dir to be specified")
+
+        stage_out_app = _zip_stage_out_app(dm)
+        app_fut = stage_out_app(zip_path, inside_path, working_dir, inputs=[file], _parsl_staging_inhibit=True, parent_fut=parent_fut)
+        return app_fut
+
+
+def _zip_stage_out(zip_file, inside_path, working_dir, parent_fut=None, inputs=[], _parsl_staging_inhibit=True):
+    file = inputs[0]
+
+    os.makedirs(os.path.dirname(zip_file), exist_ok=True)
+
+    with filelock.FileLock(zip_file + ".lock"):
+        with zipfile.ZipFile(zip_file, mode='a', compression=zipfile.ZIP_DEFLATED) as z:
+            z.write(file, arcname=inside_path)
+
+    os.remove(file)
+
+
+def _zip_stage_out_app(dm):
+    return parsl.python_app(executors=['_parsl_internal'], data_flow_kernel=dm.dfk)(_zip_stage_out)
+
+
+def zip_path_split(path: str) -> Tuple[str, str]:
+    """Split zip: path into a zipfile name and a contained-file name.
+    """
+    index = path.find(".zip/")
+
+    zip_path = path[:index + 4]
+    inside_path = path[index + 5:]
+
+    return (zip_path, inside_path)

parsl/dataflow/dflow.py

@@ -177,10 +177,11 @@ class DataFlowKernel:
 
         # this must be set before executors are added since add_executors calls
         # job_status_poller.add_executors.
+        radio = self.monitoring.radio if self.monitoring else None
         self.job_status_poller = JobStatusPoller(strategy=self.config.strategy,
                                                  strategy_period=self.config.strategy_period,
                                                  max_idletime=self.config.max_idletime,
-                                                 dfk=self)
+                                                 monitoring=radio)
 
         self.executors: Dict[str, ParslExecutor] = {}
 
@@ -239,16 +240,29 @@ class DataFlowKernel:
         task_log_info['task_stdin'] = task_record['kwargs'].get('stdin', None)
         stdout_spec = task_record['kwargs'].get('stdout', None)
         stderr_spec = task_record['kwargs'].get('stderr', None)
-        try:
-            stdout_name, _ = get_std_fname_mode('stdout', stdout_spec)
-        except Exception as e:
-            logger.warning("Incorrect stdout format {} for Task {}".format(stdout_spec, task_record['id']))
-            stdout_name = str(e)
-        try:
-            stderr_name, _ = get_std_fname_mode('stderr', stderr_spec)
-        except Exception as e:
-            logger.warning("Incorrect stderr format {} for Task {}".format(stderr_spec, task_record['id']))
-            stderr_name = str(e)
+
+        # stdout and stderr strings are set to the filename if we can
+        # interpret the specification; otherwise, set to the empty string
+        # (on exception, or when not specified)
+
+        if stdout_spec is not None:
+            try:
+                stdout_name, _ = get_std_fname_mode('stdout', stdout_spec)
+            except Exception:
+                logger.exception("Could not parse stdout specification {} for task {}".format(stdout_spec, task_record['id']))
+                stdout_name = ""
+        else:
+            stdout_name = ""
+
+        if stderr_spec is not None:
+            try:
+                stderr_name, _ = get_std_fname_mode('stderr', stderr_spec)
+            except Exception:
+                logger.exception("Could not parse stderr specification {} for task {}".format(stderr_spec, task_record['id']))
+                stderr_name = ""
+        else:
+            stderr_name = ""
+
         task_log_info['task_stdout'] = stdout_name
         task_log_info['task_stderr'] = stderr_name
         task_log_info['task_fail_history'] = ",".join(task_record['fail_history'])
@@ -674,14 +688,6 @@ class DataFlowKernel:
     def launch_task(self, task_record: TaskRecord) -> Future:
         """Handle the actual submission of the task to the executor layer.
 
-        If the app task has the executors attributes not set (default=='all')
-        the task is launched on a randomly selected executor from the
-        list of executors. This behavior could later be updated to support
-        binding to executors based on user specified criteria.
-
-        If the app task specifies a particular set of executors, it will be
-        targeted at those specific executors.
-
         Args:
             task_record : The task record
 
@@ -714,14 +720,18 @@ class DataFlowKernel:
 
         if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
             wrapper_logging_level = logging.DEBUG if self.monitoring.monitoring_debug else logging.INFO
-            (function, args, kwargs) = monitor_wrapper(function, args, kwargs, try_id, task_id,
-                                                       self.monitoring.monitoring_hub_url,
-                                                       self.run_id,
-                                                       wrapper_logging_level,
-                                                       self.monitoring.resource_monitoring_interval,
-                                                       executor.radio_mode,
-                                                       executor.monitor_resources(),
-                                                       self.run_dir)
+            (function, args, kwargs) = monitor_wrapper(f=function,
+                                                       args=args,
+                                                       kwargs=kwargs,
+                                                       x_try_id=try_id,
+                                                       x_task_id=task_id,
+                                                       monitoring_hub_url=self.monitoring.monitoring_hub_url,
+                                                       run_id=self.run_id,
+                                                       logging_level=wrapper_logging_level,
+                                                       sleep_dur=self.monitoring.resource_monitoring_interval,
+                                                       radio_mode=executor.radio_mode,
+                                                       monitor_resources=executor.monitor_resources(),
+                                                       run_dir=self.run_dir)
 
         with self.submitter_lock:
             exec_fu = executor.submit(function, task_record['resource_specification'], *args, **kwargs)
@@ -1128,6 +1138,10 @@ class DataFlowKernel:
             executor.run_dir = self.run_dir
             executor.hub_address = self.hub_address
             executor.hub_port = self.hub_zmq_port
+            if self.monitoring:
+                executor.monitoring_radio = self.monitoring.radio
+            else:
+                executor.monitoring_radio = None
             if hasattr(executor, 'provider'):
                 if hasattr(executor.provider, 'script_dir'):
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')
@@ -1214,21 +1228,7 @@ class DataFlowKernel:
         self.job_status_poller.close()
         logger.info("Terminated job status poller")
 
-        logger.info("Scaling in and shutting down executors")
-
-        for ef in self.job_status_poller._executor_facades:
-            if not ef.executor.bad_state_is_set:
-                logger.info(f"Scaling in executor {ef.executor.label}")
-
-                # this code needs to be at least as many blocks as need
-                # cancelling, but it is safe to be more, as the scaling
-                # code will cope with being asked to cancel more blocks
-                # than exist.
-                block_count = len(ef.status)
-                ef.scale_in(block_count)
-
-            else:  # and bad_state_is_set
-                logger.warning(f"Not scaling in executor {ef.executor.label} because it is in bad state")
+        logger.info("Shutting down executors")
 
         for executor in self.executors.values():
             logger.info(f"Shutting down executor {executor.label}")
@@ -1245,8 +1245,7 @@ class DataFlowKernel:
                                   'tasks_completed_count': self.task_state_counts[States.exec_done],
                                   "time_began": self.time_began,
                                   'time_completed': self.time_completed,
-                                  'run_id': self.run_id, 'rundir': self.run_dir,
-                                  'exit_now': True})
+                                  'run_id': self.run_id, 'rundir': self.run_dir})
 
             logger.info("Terminating monitoring")
             self.monitoring.close()
@@ -1396,10 +1395,20 @@ class DataFlowKernel:
 
     @staticmethod
     def _log_std_streams(task_record: TaskRecord) -> None:
-        if task_record['app_fu'].stdout is not None:
-            logger.info("Standard output for task {} available at {}".format(task_record['id'], task_record['app_fu'].stdout))
-        if task_record['app_fu'].stderr is not None:
-            logger.info("Standard error for task {} available at {}".format(task_record['id'], task_record['app_fu'].stderr))
+        tid = task_record['id']
+
+        def log_std_stream(name: str, target) -> None:
+            if target is None:
+                logger.info(f"{name} for task {tid} will not be redirected.")
+            elif isinstance(target, str):
+                logger.info(f"{name} for task {tid} will be redirected to {target}")
+            elif isinstance(target, tuple) and len(target) == 2:
+                logger.info(f"{name} for task {tid} will be redirected to {target[0]} with mode {target[1]}")
+            else:
+                logger.error(f"{name} for task {tid} has unknown specification: {target!r}")
+
+        log_std_stream("Standard out", task_record['app_fu'].stdout)
+        log_std_stream("Standard error", task_record['app_fu'].stderr)
 
 
 class DataFlowKernelLoader:

parsl/dataflow/futures.py

@@ -1,10 +1,3 @@
-"""This module implements the AppFutures.
-
-We have two basic types of futures:
-    1. DataFutures which represent data objects
-    2. AppFutures which represent the futures on App/Leaf tasks.
-
-"""
 from __future__ import annotations
 
 from concurrent.futures import Future

parsl/executors/base.py

@@ -1,9 +1,9 @@
 from abc import ABCMeta, abstractmethod
 from concurrent.futures import Future
-from typing import Any, Callable, Dict, Optional, List
+from typing import Any, Callable, Dict, Optional
 from typing_extensions import Literal, Self
 
-from parsl.jobs.states import JobStatus
+from parsl.monitoring.radios import MonitoringRadio
 
 
 class ParslExecutor(metaclass=ABCMeta):
@@ -79,13 +79,6 @@ class ParslExecutor(metaclass=ABCMeta):
         """
         pass
 
-    def create_monitoring_info(self, status: Dict[str, JobStatus]) -> List[object]:
-        """Create a monitoring message for each block based on the poll status.
-
-        :return: a list of dictionaries mapping to the info of each block
-        """
-        return []
-
     def monitor_resources(self) -> bool:
         """Should resource monitoring happen for tasks on running on this executor?
 
@@ -135,3 +128,13 @@ class ParslExecutor(metaclass=ABCMeta):
     @hub_port.setter
     def hub_port(self, value: Optional[int]) -> None:
         self._hub_port = value
+
+    @property
+    def monitoring_radio(self) -> Optional[MonitoringRadio]:
+        """Local radio for sending monitoring messages
+        """
+        return self._monitoring_radio
+
+    @monitoring_radio.setter
+    def monitoring_radio(self, value: Optional[MonitoringRadio]) -> None:
+        self._monitoring_radio = value

parsl/executors/high_throughput/executor.py

@@ -5,7 +5,6 @@ import typeguard
 import logging
 import threading
 import queue
-import datetime
 import pickle
 from dataclasses import dataclass
 from multiprocessing import Process, Queue
@@ -18,7 +17,7 @@ import parsl.launchers
 from parsl.serialize import pack_res_spec_apply_message, deserialize
 from parsl.serialize.errors import SerializationError, DeserializationError
 from parsl.app.errors import RemoteExceptionWrapper
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobStatus, JobState, TERMINAL_STATES
 from parsl.executors.high_throughput import zmq_pipes
 from parsl.executors.high_throughput import interchange
 from parsl.executors.errors import (
@@ -677,22 +676,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
         # Return the future
         return fut
 
-    def create_monitoring_info(self, status):
-        """ Create a msg for monitoring based on the poll status
-
-        """
-        msg = []
-        for bid, s in status.items():
-            d = {}
-            d['run_id'] = self.run_id
-            d['status'] = s.status_name
-            d['timestamp'] = datetime.datetime.now()
-            d['executor_label'] = self.label
-            d['job_id'] = self.blocks_to_job_id.get(bid, None)
-            d['block_id'] = bid
-            msg.append(d)
-        return msg
-
     @property
     def workers_per_node(self) -> Union[int, float]:
         return self._workers_per_node
@@ -730,8 +713,20 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
             tasks: int  # sum of tasks in this block
             idle: float  # shortest idle time of any manager in this block
 
+        # block_info will be populated from two sources:
+        # the Job Status Poller mutable block list, and the list of blocks
+        # which have connected to the interchange.
+
+        def new_block_info():
+            return BlockInfo(tasks=0, idle=float('inf'))
+
+        block_info: Dict[str, BlockInfo] = defaultdict(new_block_info)
+
+        for block_id, job_status in self._status.items():
+            if job_status.state not in TERMINAL_STATES:
+                block_info[block_id] = new_block_info()
+
         managers = self.connected_managers()
-        block_info: Dict[str, BlockInfo] = defaultdict(lambda: BlockInfo(tasks=0, idle=float('inf')))
         for manager in managers:
             if not manager['active']:
                 continue

parsl/executors/high_throughput/process_worker_pool.py

@@ -361,7 +361,9 @@ class Manager:
                     kill_event.set()
                 else:
                     task_recv_counter += len(tasks)
-                    logger.debug("Got executor tasks: {}, cumulative count of tasks: {}".format([t['task_id'] for t in tasks], task_recv_counter))
+                    logger.debug("Got executor tasks: {}, cumulative count of tasks: {}".format(
+                        [t['task_id'] for t in tasks], task_recv_counter
+                    ))
 
                     for task in tasks:
                         self.task_scheduler.put_task(task)

parsl/executors/status_handling.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
+import datetime
 import logging
 import threading
+import time
 from itertools import compress
 from abc import abstractmethod, abstractproperty
 from concurrent.futures import Future
-from typing import List, Any, Dict, Optional, Tuple, Union, Callable
+from typing import List, Any, Dict, Optional, Sequence, Tuple, Union, Callable
 
 from parsl.executors.base import ParslExecutor
 from parsl.executors.errors import BadStateException, ScalingFailed
 from parsl.jobs.states import JobStatus, JobState
 from parsl.jobs.error_handlers import simple_error_handler, noop_error_handler
+from parsl.monitoring.message_type import MessageType
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import AtomicIDCounter
 
@@ -71,6 +74,9 @@ class BlockProviderExecutor(ParslExecutor):
         self.blocks_to_job_id = {}  # type: Dict[str, str]
         self.job_ids_to_block = {}  # type: Dict[str, str]
 
+        self._last_poll_time = 0.0
+        self._status = {}  # type: Dict[str, JobStatus]
+
     def _make_status_dict(self, block_ids: List[str], status_list: List[JobStatus]) -> Dict[str, JobStatus]:
         """Given a list of block ids and a list of corresponding status strings,
         returns a dictionary mapping each block id to the corresponding status
@@ -102,12 +108,6 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: str, message: str):
-        """Marks a job that has failed to start but would not otherwise be included in status()
-        as failed and report it in status()
-        """
-        self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
-
     @abstractproperty
     def outstanding(self) -> int:
         """This should return the number of tasks that the executor has been given to run (waiting to run, and running now)"""
@@ -198,8 +198,7 @@ class BlockProviderExecutor(ParslExecutor):
                 self.job_ids_to_block[job_id] = block_id
                 block_ids.append(block_id)
             except Exception as ex:
-                self._fail_job_async(block_id,
-                                     "Failed to start block {}: {}".format(block_id, ex))
+                self._simulated_status[block_id] = JobStatus(JobState.FAILED, "Failed to start block {}: {}".format(block_id, ex))
         return block_ids
 
     @abstractmethod
@@ -241,3 +240,77 @@ class BlockProviderExecutor(ParslExecutor):
     @abstractproperty
     def workers_per_node(self) -> Union[int, float]:
         pass
+
+    def send_monitoring_info(self, status: Dict) -> None:
+        # Send monitoring info for HTEX when monitoring enabled
+        if self.monitoring_radio:
+            msg = self.create_monitoring_info(status)
+            logger.debug("Sending message {} to hub from job status poller".format(msg))
+            self.monitoring_radio.send((MessageType.BLOCK_INFO, msg))
+
+    def create_monitoring_info(self, status: Dict[str, JobStatus]) -> Sequence[object]:
+        """Create a monitoring message for each block based on the poll status.
+        """
+        msg = []
+        for bid, s in status.items():
+            d: Dict[str, Any] = {}
+            d['run_id'] = self.run_id
+            d['status'] = s.status_name
+            d['timestamp'] = datetime.datetime.now()
+            d['executor_label'] = self.label
+            d['job_id'] = self.blocks_to_job_id.get(bid, None)
+            d['block_id'] = bid
+            msg.append(d)
+        return msg
+
+    def poll_facade(self) -> None:
+        now = time.time()
+        if now >= self._last_poll_time + self.status_polling_interval:
+            previous_status = self._status
+            self._status = self.status()
+            self._last_poll_time = now
+            delta_status = {}
+            for block_id in self._status:
+                if block_id not in previous_status \
+                   or previous_status[block_id].state != self._status[block_id].state:
+                    delta_status[block_id] = self._status[block_id]
+
+            if delta_status:
+                self.send_monitoring_info(delta_status)
+
+    @property
+    def status_facade(self) -> Dict[str, JobStatus]:
+        """Return the status of all jobs/blocks of the executor of this poller.
+
+        :return: a dictionary mapping block ids (in string) to job status
+        """
+        return self._status
+
+    def scale_in_facade(self, n: int, max_idletime: Optional[float] = None) -> List[str]:
+
+        if max_idletime is None:
+            block_ids = self.scale_in(n)
+        else:
+            # This is a HighThroughputExecutor-specific interface violation.
+            # This code hopes, through pan-codebase reasoning, that this
+            # scale_in method really does come from HighThroughputExecutor,
+            # and so does have an extra max_idletime parameter not present
+            # in the executor interface.
+            block_ids = self.scale_in(n, max_idletime=max_idletime)  # type: ignore[call-arg]
+        if block_ids is not None:
+            new_status = {}
+            for block_id in block_ids:
+                new_status[block_id] = JobStatus(JobState.CANCELLED)
+                del self._status[block_id]
+            self.send_monitoring_info(new_status)
+        return block_ids
+
+    def scale_out_facade(self, n: int) -> List[str]:
+        block_ids = self.scale_out(n)
+        if block_ids is not None:
+            new_status = {}
+            for block_id in block_ids:
+                new_status[block_id] = JobStatus(JobState.PENDING)
+            self.send_monitoring_info(new_status)
+            self._status.update(new_status)
+        return block_ids

parsl/executors/taskvine/executor.py

@@ -596,7 +596,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return 1
 
-    def scale_in(self, count):
+    def scale_in(self, count: int) -> List[str]:
         """Scale in method. Cancel a given number of blocks
         """
         # Obtain list of blocks to kill
@@ -605,9 +605,14 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # Cancel the blocks provisioned
         if self.provider:
-            self.provider.cancel(kill_ids)
+            logger.info(f"Scaling in jobs: {kill_ids}")
+            r = self.provider.cancel(kill_ids)
+            job_ids = self._filter_scale_in_ids(kill_ids, r)
+            block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
+            return block_ids_killed
         else:
             logger.error("No execution provider available to scale")
+            return []
 
     def shutdown(self, *args, **kwargs):
         """Shutdown the executor. Sets flag to cancel the submit process and

parsl/executors/workqueue/executor.py

@@ -691,7 +691,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return 1
 
-    def scale_in(self, count):
+    def scale_in(self, count: int) -> List[str]:
         """Scale in method.
         """
         # Obtain list of blocks to kill
@@ -700,9 +700,14 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # Cancel the blocks provisioned
         if self.provider:
-            self.provider.cancel(kill_ids)
+            logger.info(f"Scaling in jobs: {kill_ids}")
+            r = self.provider.cancel(kill_ids)
+            job_ids = self._filter_scale_in_ids(kill_ids, r)
+            block_ids_killed = [self.job_ids_to_block[jid] for jid in job_ids]
+            return block_ids_killed
         else:
-            logger.error("No execution provider available to scale")
+            logger.error("No execution provider available to scale in")
+            return []
 
     def shutdown(self, *args, **kwargs):
         """Shutdown the executor. Sets flag to cancel the submit process and