toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/batchSystems/abstractBatchSystem.py

@@ -18,18 +18,10 @@ import shutil
 import time
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser, _ArgumentGroup
+from collections.abc import Iterator
 from contextlib import contextmanager
 from threading import Condition
-from typing import (Any,
-                    ContextManager,
-                    Dict,
-                    Iterator,
-                    List,
-                    NamedTuple,
-                    Optional,
-                    Set,
-                    Union,
-                    cast)
+from typing import Any, ContextManager, NamedTuple, Optional, Union, cast
 
 from toil.batchSystems.options import OptionSetter
 from toil.bus import MessageBus, MessageOutbox
@@ -45,19 +37,41 @@ logger = logging.getLogger(__name__)
 # Value to use as exitStatus in UpdatedBatchJobInfo.exitStatus when status is not available.
 EXIT_STATUS_UNAVAILABLE_VALUE = 255
 
+
 class BatchJobExitReason(enum.IntEnum):
-    FINISHED: int = 1
+    FINISHED = 1
     """Successfully finished."""
-    FAILED: int = 2
+    FAILED = 2
     """Job finished, but failed."""
-    LOST: int = 3
+    LOST = 3
     """Preemptable failure (job's executing host went away)."""
-    KILLED: int = 4
+    KILLED = 4
     """Job killed before finishing."""
-    ERROR: int = 5
+    ERROR = 5
     """Internal error."""
-    MEMLIMIT: int = 6
+    MEMLIMIT = 6
     """Job hit batch system imposed memory limit."""
+    MISSING = 7
+    """Job disappeared from the scheduler without actually stopping, so Toil killed it."""
+    MAXJOBDURATION = 8
+    """Job ran longer than --maxJobDuration, so Toil killed it."""
+    PARTITION = 9
+    """Job was not able to talk to the leader via the job store, so Toil declared it failed."""
+
+    @classmethod
+    def to_string(cls, value: int) -> str:
+        """
+        Convert to human-readable string.
+
+        Given an int that may be or may be equal to a value from the enum,
+        produce the string value of its matching enum entry, or a stringified
+        int.
+        """
+        try:
+            return cls(value).name
+        except ValueError:
+            return str(value)
+
 
 class UpdatedBatchJobInfo(NamedTuple):
     jobID: int
@@ -65,12 +79,14 @@ class UpdatedBatchJobInfo(NamedTuple):
     """
     The exit status (integer value) of the job. 0 implies successful.
 
-    EXIT_STATUS_UNAVAILABLE_VALUE is used when the exit status is not available (e.g. job is lost).
+    EXIT_STATUS_UNAVAILABLE_VALUE is used when the exit status is not available
+    (e.g. job is lost, or otherwise died but actual exit code was not reported).
     """
 
     exitReason: Optional[BatchJobExitReason]
     wallTime: Union[float, int, None]
 
+
 # Information required for worker cleanup on shutdown of the batch system.
 class WorkerCleanupInfo(NamedTuple):
     work_dir: Optional[str]
@@ -88,8 +104,10 @@ class WorkerCleanupInfo(NamedTuple):
     'onSuccess', 'onError', 'never')
     """
 
+
 class AbstractBatchSystem(ABC):
     """An abstract base class to represent the interface the batch system must provide to Toil."""
+
     @classmethod
     @abstractmethod
     def supportsAutoDeployment(cls) -> bool:
@@ -141,22 +159,29 @@ class AbstractBatchSystem(ABC):
         """
 
     @abstractmethod
-    def issueBatchJob(self, jobDesc: JobDescription, job_environment: Optional[Dict[str, str]] = None) -> int:
+    def issueBatchJob(
+        self,
+        command: str,
+        job_desc: JobDescription,
+        job_environment: Optional[dict[str, str]] = None,
+    ) -> int:
         """
         Issues a job with the specified command to the batch system and returns
-        a unique jobID.
+        a unique job ID number.
 
-        :param jobDesc: a toil.job.JobDescription
+        :param command: the command to execute somewhere to run the Toil
+            worker process
+        :param job_desc: the JobDescription for the job being run
         :param job_environment: a collection of job-specific environment
-                                variables to be set on the worker.
+            variables to be set on the worker.
 
-        :return: a unique jobID that can be used to reference the newly issued
-                 job
+        :return: a unique job ID number that can be used to reference the newly
+            issued job
         """
         raise NotImplementedError()
 
     @abstractmethod
-    def killBatchJobs(self, jobIDs: List[int]) -> None:
+    def killBatchJobs(self, jobIDs: list[int]) -> None:
         """
         Kills the given job IDs. After returning, the killed jobs will not
         appear in the results of getRunningBatchJobIDs. The killed job will not
@@ -169,24 +194,24 @@ class AbstractBatchSystem(ABC):
     # FIXME: Return value should be a set (then also fix the tests)
 
     @abstractmethod
-    def getIssuedBatchJobIDs(self) -> List[int]:
+    def getIssuedBatchJobIDs(self) -> list[int]:
         """
         Gets all currently issued jobs
 
-        :return: A list of jobs (as jobIDs) currently issued (may be running, or may be
-                 waiting to be run). Despite the result being a list, the ordering should not
-                 be depended upon.
+        :return: A list of jobs (as job ID numbers) currently issued (may be
+            running, or may be waiting to be run). Despite the result being a
+            list, the ordering should not be depended upon.
         """
         raise NotImplementedError()
 
     @abstractmethod
-    def getRunningBatchJobIDs(self) -> Dict[int, float]:
+    def getRunningBatchJobIDs(self) -> dict[int, float]:
         """
-        Gets a map of jobs as jobIDs that are currently running (not just waiting)
-        and how long they have been running, in seconds.
+        Gets a map of jobs as job ID numbers that are currently running (not
+        just waiting) and how long they have been running, in seconds.
 
-        :return: dictionary with currently running jobID keys and how many seconds they have
-                 been running as the value
+        :return: dictionary with currently running job ID number keys and how
+            many seconds they have been running as the value
         """
         raise NotImplementedError()
 
@@ -268,7 +293,7 @@ class AbstractBatchSystem(ABC):
             returning nothing, used to update run configuration as a side effect.
         """
 
-    def getWorkerContexts(self) -> List[ContextManager[Any]]:
+    def getWorkerContexts(self) -> list[ContextManager[Any]]:
         """
         Get a list of picklable context manager objects to wrap worker work in,
         in order.
@@ -284,7 +309,9 @@ class AbstractBatchSystem(ABC):
 class BatchSystemSupport(AbstractBatchSystem):
     """Partial implementation of AbstractBatchSystem, support methods."""
 
-    def __init__(self, config: Config, maxCores: float, maxMemory: int, maxDisk: int) -> None:
+    def __init__(
+        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+    ) -> None:
         """
         Initialize initial state of the object.
 
@@ -306,7 +333,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         self.maxCores = maxCores
         self.maxMemory = maxMemory
         self.maxDisk = maxDisk
-        self.environment: Dict[str, str] = {}
+        self.environment: dict[str, str] = {}
         if config.workflowID is None:
             raise Exception("config.workflowID must be set")
         else:
@@ -332,9 +359,11 @@ class BatchSystemSupport(AbstractBatchSystem):
                greater than allowed
         """
         try:
-            for resource, requested, available in [('cores', requirer.cores, self.maxCores),
-                                                   ('memory', requirer.memory, self.maxMemory),
-                                                   ('disk', requirer.disk, self.maxDisk)]:
+            for resource, requested, available in [
+                ("cores", requirer.cores, self.maxCores),
+                ("memory", requirer.memory, self.maxMemory),
+                ("disk", requirer.disk, self.maxDisk),
+            ]:
                 assert requested is not None
                 if requested > available:
                     raise InsufficientSystemResources(requirer, resource, available)
@@ -343,7 +372,7 @@ class BatchSystemSupport(AbstractBatchSystem):
         except InsufficientSystemResources as e:
             # Add more annotation info to the error
             e.batch_system = self.__class__.__name__ or None
-            e.source = self.config.workDir if e.resource == 'disk' else None
+            e.source = self.config.workDir if e.resource == "disk" else None
             raise e
 
     def _check_accelerator_request(self, requirer: Requirer) -> None:
@@ -356,9 +385,12 @@ class BatchSystemSupport(AbstractBatchSystem):
         """
         if len(requirer.accelerators) > 0:
             # By default we assume we can't fulfill any of these
-            raise InsufficientSystemResources(requirer, 'accelerators', [], details=[
-                'The batch system does not support any accelerators.'
-            ])
+            raise InsufficientSystemResources(
+                requirer,
+                "accelerators",
+                [],
+                details=["The batch system does not support any accelerators."],
+            )
 
     def setEnv(self, name: str, value: Optional[str] = None) -> None:
         """
@@ -415,7 +447,9 @@ class BatchSystemSupport(AbstractBatchSystem):
         # And if nothing is specified use the workDir.
         return Toil.getToilWorkDir(self.config.workDir)
 
-    def format_std_out_err_path(self, toil_job_id: int, cluster_job_id: str, std: str) -> str:
+    def format_std_out_err_path(
+        self, toil_job_id: int, cluster_job_id: str, std: str
+    ) -> str:
         """
         Format path for batch system standard output/error and other files
         generated by the batch system itself.
@@ -434,18 +468,20 @@ class BatchSystemSupport(AbstractBatchSystem):
         if self.config.noStdOutErr:
             return os.devnull
 
-        file_name: str = f'toil_{self.config.workflowID}.{toil_job_id}.{cluster_job_id}.{std}.log'
+        file_name: str = (
+            f"toil_{self.config.workflowID}.{toil_job_id}.{cluster_job_id}.{std}.log"
+        )
         logs_dir: str = self.get_batch_logs_dir()
         return os.path.join(logs_dir, file_name)
-    
+
     def format_std_out_err_glob(self, toil_job_id: int) -> str:
         """
         Get a glob string that will match all file paths generated by format_std_out_err_path for a job.
         """
-        file_glob: str = f'toil_{self.config.workflowID}.{toil_job_id}.*.log'
+        file_glob: str = f"toil_{self.config.workflowID}.{toil_job_id}.*.log"
         logs_dir: str = self.get_batch_logs_dir()
         return os.path.join(logs_dir, file_glob)
-        
+
     @staticmethod
     def workerCleanup(info: WorkerCleanupInfo) -> None:
         """
@@ -456,23 +492,28 @@ class BatchSystemSupport(AbstractBatchSystem):
         :param WorkerCleanupInfo info: A named tuple consisting of all the relevant information
                for cleaning up the worker.
         """
-        logger.debug('Attempting worker cleanup')
+        logger.debug("Attempting worker cleanup")
         assert isinstance(info, WorkerCleanupInfo)
         assert info.workflow_id is not None
         workflowDir = Toil.getLocalWorkflowDir(info.workflow_id, info.work_dir)
-        coordination_dir = Toil.get_local_workflow_coordination_dir(info.workflow_id, info.work_dir, info.coordination_dir)
+        coordination_dir = Toil.get_local_workflow_coordination_dir(
+            info.workflow_id, info.work_dir, info.coordination_dir
+        )
         DeferredFunctionManager.cleanupWorker(coordination_dir)
         workflowDirContents = os.listdir(workflowDir)
-        AbstractFileStore.shutdownFileStore(info.workflow_id, info.work_dir, info.coordination_dir)
-        if info.clean_work_dir in ('always', 'onSuccess', 'onError'):
+        AbstractFileStore.shutdownFileStore(
+            info.workflow_id, info.work_dir, info.coordination_dir
+        )
+        if info.clean_work_dir in ("always", "onSuccess", "onError"):
             if workflowDirContents in ([], [cacheDirName(info.workflow_id)]):
-                logger.debug('Deleting workflow directory %s', workflowDir)
+                logger.debug("Deleting workflow directory %s", workflowDir)
                 shutil.rmtree(workflowDir, ignore_errors=True)
             if coordination_dir != workflowDir:
                 # No more coordination to do here either.
-                logger.debug('Deleting coordination directory %s', coordination_dir)
+                logger.debug("Deleting coordination directory %s", coordination_dir)
                 shutil.rmtree(coordination_dir, ignore_errors=True)
 
+
 class NodeInfo:
     """
     The coresUsed attribute  is a floating point value between 0 (all cores idle) and 1 (all cores
@@ -489,10 +530,17 @@ class NodeInfo:
     The workers attribute is an integer reflecting the number of workers currently active workers
     on the node.
     """
-    def __init__(self, coresUsed: float, memoryUsed: float,
-                 coresTotal: float, memoryTotal: int,
-                 requestedCores: float, requestedMemory: int,
-                 workers: int) -> None:
+
+    def __init__(
+        self,
+        coresUsed: float,
+        memoryUsed: float,
+        coresTotal: float,
+        memoryTotal: int,
+        requestedCores: float,
+        requestedMemory: int,
+        workers: int,
+    ) -> None:
         self.coresUsed = coresUsed
         self.memoryUsed = memoryUsed
 
@@ -509,13 +557,15 @@ class AbstractScalableBatchSystem(AbstractBatchSystem):
     """
     A batch system that supports a variable number of worker nodes.
 
-    Used by :class:`toil.provisioners.clusterScaler.ClusterScaler` 
+    Used by :class:`toil.provisioners.clusterScaler.ClusterScaler`
     to scale the number of worker nodes in the cluster
     up or down depending on overall load.
     """
 
     @abstractmethod
-    def getNodes(self, preemptible: Optional[bool] = None, timeout: int = 600) -> Dict[str, NodeInfo]:
+    def getNodes(
+        self, preemptible: Optional[bool] = None, timeout: int = 600
+    ) -> dict[str, NodeInfo]:
         """
         Returns a dictionary mapping node identifiers of preemptible or non-preemptible nodes to
         NodeInfo objects, one for each node.
@@ -560,7 +610,15 @@ class AbstractScalableBatchSystem(AbstractBatchSystem):
 
 
 class InsufficientSystemResources(Exception):
-    def __init__(self, requirer: Requirer, resource: str, available: Optional[ParsedRequirement] = None, batch_system: Optional[str] = None, source: Optional[str] = None, details: List[str] = []) -> None:
+    def __init__(
+        self,
+        requirer: Requirer,
+        resource: str,
+        available: Optional[ParsedRequirement] = None,
+        batch_system: Optional[str] = None,
+        source: Optional[str] = None,
+        details: list[str] = [],
+    ) -> None:
         """
         Make a new exception about how we couldn't get enough of something.
 
@@ -573,7 +631,7 @@ class InsufficientSystemResources(Exception):
         :param details: Any extra details about the problem that can be attached to the error.
         """
 
-        self.job_name : Optional[str] = str(requirer)
+        self.job_name: Optional[str] = str(requirer)
         self.resource = resource
         self.requested = cast(ParsedRequirement, getattr(requirer, resource))
         self.available = available
@@ -586,38 +644,52 @@ class InsufficientSystemResources(Exception):
         Explain the exception.
         """
 
-        unit = 'bytes of ' if self.resource in ('disk', 'memory') else ''
-        purpose = ' for temporary space' if self.resource == 'disk' else ''
-        qualifier = ' free on {self.source}' if self.resource == 'disk' and self.source is not None else ''
+        unit = "bytes of " if self.resource in ("disk", "memory") else ""
+        purpose = " for temporary space" if self.resource == "disk" else ""
+        qualifier = (
+            " free on {self.source}"
+            if self.resource == "disk" and self.source is not None
+            else ""
+        )
 
         msg = []
         if self.job_name is not None:
-            msg.append(f'The job {self.job_name} is requesting ')
+            msg.append(f"The job {self.job_name} is requesting ")
         else:
-            msg.append(f'Requesting ')
-        msg.append(f'{self.requested} {unit}{self.resource}')
+            msg.append(f"Requesting ")
+        msg.append(f"{self.requested} {unit}{self.resource}")
         msg.append(purpose)
         if self.available is not None:
-            msg.append(f', more than the maximum of {self.available} {unit}{self.resource}{qualifier} that {self.batch_system or "this batch system"} was configured with')
-            if self.resource in ('cores', 'memory', 'disk'):
-                msg.append(f', or enforced by --max{self.resource.capitalize()}')
+            msg.append(
+                f', more than the maximum of {self.available} {unit}{self.resource}{qualifier} that {self.batch_system or "this batch system"} was configured with'
+            )
+            if self.resource in ("cores", "memory", "disk"):
+                msg.append(f", or enforced by --max{self.resource.capitalize()}")
         else:
-            msg.append(', but that is not available')
-        msg.append('.')
+            msg.append(", but that is not available")
+        msg.append(".")
 
-        if self.resource == 'disk':
-            msg.append(' Try setting/changing the toil option "--workDir" or changing the base temporary directory by setting TMPDIR.')
+        if self.resource == "disk":
+            msg.append(
+                ' Try setting/changing the toil option "--workDir" or changing the base temporary directory by setting TMPDIR.'
+            )
 
         for detail in self.details:
-            msg.append(' ')
+            msg.append(" ")
             msg.append(detail)
 
-        return ''.join(msg)
+        return "".join(msg)
 
 
 class AcquisitionTimeoutException(Exception):
     """To be raised when a resource request times out."""
-    def __init__(self, resource: str, requested: Union[int, float, Set[int]], available: Union[int, float, Set[int]]) -> None:
+
+    def __init__(
+        self,
+        resource: str,
+        requested: Union[int, float, set[int]],
+        available: Union[int, float, set[int]],
+    ) -> None:
         """
         Creates an instance of this exception that indicates which resource is insufficient for
         current demands, as well as the resources requested and actually available.
@@ -637,7 +709,10 @@ class ResourcePool:
     Provides a context manager to do something with an amount of resource
     acquired.
     """
-    def __init__(self, initial_value: int, resource_type: str, timeout: float = 5) -> None:
+
+    def __init__(
+        self, initial_value: int, resource_type: str, timeout: float = 5
+    ) -> None:
         super().__init__()
         # We use this condition to signal everyone whenever some resource is released.
         # We use its associated lock to guard value.
@@ -671,8 +746,11 @@ class ResourcePool:
             while amount > self.value:
                 if time.time() - startTime >= self.timeout:
                     # This means the thread timed out waiting for the resource.
-                    raise AcquisitionTimeoutException(resource=self.resource_type,
-                                                      requested=amount, available=self.value)
+                    raise AcquisitionTimeoutException(
+                        resource=self.resource_type,
+                        requested=amount,
+                        available=self.value,
+                    )
                 # Allow self.timeout seconds to get the resource, else quit
                 # through the above if condition. This wait + timeout is the
                 # last thing in the loop such that a request that takes longer
@@ -713,7 +791,10 @@ class ResourceSet:
     Provides a context manager to do something with a set of of resources
     acquired.
     """
-    def __init__(self, initial_value: Set[int], resource_type: str, timeout: float = 5) -> None:
+
+    def __init__(
+        self, initial_value: set[int], resource_type: str, timeout: float = 5
+    ) -> None:
         super().__init__()
         # We use this condition to signal everyone whenever some resource is released.
         # We use its associated lock to guard value.
@@ -723,7 +804,7 @@ class ResourceSet:
         self.resource_type = resource_type
         self.timeout = timeout
 
-    def acquireNow(self, subset: Set[int]) -> bool:
+    def acquireNow(self, subset: set[int]) -> bool:
         """
         Reserve the given amount of the given resource.
         Returns True if successful and False if this is not possible immediately.
@@ -735,7 +816,7 @@ class ResourceSet:
             self.value -= subset
             return True
 
-    def acquire(self, subset: Set[int]) -> None:
+    def acquire(self, subset: set[int]) -> None:
         """
         Reserve the given amount of the given resource.
         Raises AcquisitionTimeoutException if this is not possible in under
@@ -746,8 +827,11 @@ class ResourceSet:
             while subset > self.value:
                 if time.time() - startTime >= self.timeout:
                     # This means the thread timed out waiting for the resource.
-                    raise AcquisitionTimeoutException(resource=self.resource_type,
-                                                      requested=subset, available=self.value)
+                    raise AcquisitionTimeoutException(
+                        resource=self.resource_type,
+                        requested=subset,
+                        available=self.value,
+                    )
                 # Allow self.timeout seconds to get the resource, else quit
                 # through the above if condition. This wait + timeout is the
                 # last thing in the loop such that a request that takes longer
@@ -756,12 +840,12 @@ class ResourceSet:
                 self.condition.wait(timeout=self.timeout)
             self.value -= subset
 
-    def release(self, subset: Set[int]) -> None:
+    def release(self, subset: set[int]) -> None:
         with self.condition:
             self.value |= subset
             self.condition.notify_all()
 
-    def get_free_snapshot(self) -> Set[int]:
+    def get_free_snapshot(self) -> set[int]:
         """
         Get a snapshot of what items are free right now.
         May be stale as soon as you get it, but you will need some kind of hint
@@ -776,7 +860,7 @@ class ResourceSet:
         return "ResourceSet(%s)" % self.value
 
     @contextmanager
-    def acquisitionOf(self, subset: Set[int]) -> Iterator[None]:
+    def acquisitionOf(self, subset: set[int]) -> Iterator[None]:
         self.acquire(subset)
         try:
             yield