toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/cwl/utils.py

@@ -20,11 +20,26 @@ import posixpath
 import stat
 from collections.abc import Iterable, MutableMapping, MutableSequence
 from pathlib import PurePosixPath
-from typing import Any, Callable, TypeVar, Union
-
+from typing import (
+    Any,
+    Callable,
+    TypeVar,
+    Union,
+    Optional,
+    cast,
+    MutableSequence,
+    MutableMapping,
+    TYPE_CHECKING,
+)
+from urllib.parse import unquote, urlparse
+
+if TYPE_CHECKING:
+    # This module needs to be importable even if cwltool is not installed.
+    from cwltool.utils import CWLObjectType, CWLOutputType
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.jobStores.abstractJobStore import AbstractJobStore
+from toil.lib.url import URLAccess
 
 logger = logging.getLogger(__name__)
 
@@ -208,7 +223,7 @@ def download_structure(
                 )
             else:
                 # We need to download from some other kind of URL.
-                size, executable = AbstractJobStore.read_from_url(
+                size, executable = URLAccess.read_from_url(
                     value, open(dest_path, "wb")
                 )
                 if executable:
@@ -219,3 +234,88 @@ def download_structure(
             # TODO: why?
             index[dest_path] = value
             existing[value] = dest_path
+
+
+def trim_mounts_op_down(file_or_directory: "CWLObjectType") -> None:
+    """
+    No-op function for mount-point trimming.
+    """
+    return
+
+
+def sniff_location(file_or_directory: "CWLObjectType") -> Optional[str]:
+    """
+    Get the local bare path for a CWL file or directory, or None.
+
+    :return: None if we don't have a local path or file URI
+    """
+    if file_or_directory.get('location') is None and file_or_directory.get('path') is None:
+        # file or directory is defined by contents or listing respectively, this is not redundant
+        return None
+    # Since we only consider mountable paths, if path is not file URI or bare path, don't consider it
+    path_or_url = cast(str, file_or_directory.get('location') or file_or_directory.get('path'))
+    parsed = urlparse(path_or_url)
+    if parsed.scheme == 'file':
+        return unquote(parsed.path)
+    elif parsed.scheme == '':
+        return path_or_url
+    else:
+        return None
+
+
+def trim_mounts_op_up(file_or_directory: "CWLObjectType", op_down_ret: None, child_results: list[bool]) -> bool:
+    """
+    Remove subtrees of the CWL file or directory object tree that only have redundant stuff in them.
+
+    Nonredundant for something in a directory means its path or location is not within the parent directory or doesn't match its basename
+    Nonredundant for something in a secondary file means its path or location is not adjacent to the primary file or doesn't match its basename
+
+    If on a File:
+    Returns True if anything in secondary files is nonredundant or has nonredundant children to this file, false otherwise
+    If on a Directory:
+    Returns True if anything in top level listing is nonredundant or has nonredundant children, otherwise false.
+    If something in the listing is redundant and all children are redundant, then delete it
+    :param file_or_directory: CWL file or CWL directory type
+    :return: boolean
+    """
+    own_path = sniff_location(file_or_directory)
+    if own_path is None:
+        return True
+    # basename should be set as we are the implementation
+    own_basename = cast(str, file_or_directory['basename'])
+
+    # If the basename does not match the path, then this is nonredundant
+    if not own_path.endswith("/" + own_basename):
+        return True
+
+    if file_or_directory['class'] == 'File':
+        if any(child_results):
+            # one of the children was detected as not redundant
+            return True
+        for secondary in cast(MutableSequence[MutableMapping[str, "CWLOutputType"]], file_or_directory.get('secondaryFiles', [])):
+            # secondary files should already be flagged nonredundant if they don't have either a path or location
+            secondary_path = sniff_location(secondary)
+            secondary_basename = cast(str, secondary['basename'])
+            # If we swap the secondary basename for the primary basename in the primary path, and they don't match, then they are nonredundant
+            if os.path.join(own_path[:-len(own_basename)], secondary_basename) != secondary_path:
+                return True
+    else:
+        listings = cast(MutableSequence[MutableMapping[str, "CWLOutputType"]], file_or_directory.get('listing', []))
+        if len(listings) == 0:
+            return False
+        # We assume child_results is in the same order as the directory listing
+        # iterate backwards to avoid iteration issues
+        for i in range(len(listings) - 1, -1, -1):
+            if child_results[i] is False:
+                if os.path.join(own_path, cast(str, listings[i]['basename'])) == sniff_location(listings[i]):
+                    del listings[i]
+        # If one of the listings was nonredundant, then this directory is also nonredundant
+        if any(child_results):
+            return True
+    return False
+
+def remove_redundant_mounts(cwljob: "CWLObjectType") -> None:
+    """
+    Remove any redundant mount points from the listing. Modifies the CWL object in place.
+    """
+    visit_cwl_class_and_reduce(cwljob, ["Directory", "File"], trim_mounts_op_down, trim_mounts_op_up)

toil/fileStores/__init__.py

@@ -28,7 +28,7 @@ class FileID(str):
     the job store if unavailable in the ID.
     """
 
-    def __new__(cls, fileStoreID: str, *args: Any) -> "FileID":
+    def __new__(cls, fileStoreID: str, *args: Any, **kwargs: dict[str, Any]) -> "FileID":
         return super().__new__(cls, fileStoreID)
 
     def __init__(self, fileStoreID: str, size: int, executable: bool = False) -> None:

toil/fileStores/abstractFileStore.py

@@ -671,13 +671,16 @@ class AbstractFileStore(ABC):
         Send a logging message to the leader. The message will also be \
         logged by the worker at the same level.
 
+        Does not depend on the commit system, so this is safe to use during an
+        ansynchronous commit, or without a commit afterward.
+
         :param text: The string to log.
         :param level: The logging level.
         """
-        logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
+        logger.log(level=level, msg=("LOG-TO-LEADER: " + text))
         self.logging_messages.append(dict(text=text, level=level))
 
-    @deprecated(new_function_name="export_file")
+    @deprecated(new_function_name="log_to_leader")
     def logToMaster(self, text: str, level: int = logging.INFO) -> None:
         self.log_to_leader(text, level)
 

toil/fileStores/cachingFileStore.py

@@ -1207,7 +1207,7 @@ class CachingFileStore(AbstractFileStore):
             # its temp dir and database entry.
             self._deallocateSpaceForJob()
 
-    def writeGlobalFile(self, localFileName, cleanup=False, executable=False):
+    def writeGlobalFile(self, localFileName, cleanup=False):
         """
         Creates a file in the jobstore and returns a FileID reference.
         """

toil/job.py

@@ -236,16 +236,16 @@ def parse_accelerator(
     {'count': 1, 'kind': 'gpu'}
 
     >>> parse_accelerator("nvidia-tesla-k80")
-    {'count': 1, 'kind': 'gpu', 'brand': 'nvidia', 'model': 'nvidia-tesla-k80'}
+    {'count': 1, 'kind': 'gpu', 'model': 'nvidia-tesla-k80', 'brand': 'nvidia'}
 
     >>> parse_accelerator("nvidia-tesla-k80:2")
-    {'count': 2, 'kind': 'gpu', 'brand': 'nvidia', 'model': 'nvidia-tesla-k80'}
+    {'count': 2, 'kind': 'gpu', 'model': 'nvidia-tesla-k80', 'brand': 'nvidia'}
 
     >>> parse_accelerator("gpu")
     {'count': 1, 'kind': 'gpu'}
 
     >>> parse_accelerator("cuda:1")
-    {'count': 1, 'kind': 'gpu', 'brand': 'nvidia', 'api': 'cuda'}
+    {'count': 1, 'kind': 'gpu', 'api': 'cuda', 'brand': 'nvidia'}
 
     >>> parse_accelerator({"kind": "gpu"})
     {'count': 1, 'kind': 'gpu'}
@@ -581,8 +581,8 @@ class Requirer:
         >>> Requirer._parseResource('cores', 1), Requirer._parseResource('disk', 1), \
         Requirer._parseResource('memory', 1)
         (1, 1, 1)
-        >>> Requirer._parseResource('cores', '1G'), Requirer._parseResource('disk', '1G'), \
-        Requirer._parseResource('memory', '1G')
+        >>> Requirer._parseResource('cores', '1Gi'), Requirer._parseResource('disk', '1Gi'), \
+        Requirer._parseResource('memory', '1Gi')
         (1073741824, 1073741824, 1073741824)
         >>> Requirer._parseResource('cores', 1.1)
         1.1
@@ -813,7 +813,6 @@ class JobDescription(Requirer):
     Subclassed into variants for checkpoint jobs and service jobs that have
     their specific parameters.
     """
-
     def __init__(
         self,
         requirements: Mapping[str, Union[int, str, float, bool, list]],
@@ -3146,9 +3145,8 @@ class Job:
 
         Will modify the job's description with changes that need to be committed back to the JobStore.
         """
-        if stats is not None:
-            startTime = time.time()
-            startClock = ResourceMonitor.get_total_cpu_time()
+        startTime = time.time()
+        startClock = ResourceMonitor.get_total_cpu_time()
         baseDir = os.getcwd()
 
         succeeded = False
@@ -3180,18 +3178,36 @@ class Job:
             # Change dir back to cwd dir, if changed by job (this is a safety issue)
             if os.getcwd() != baseDir:
                 os.chdir(baseDir)
+            
+            totalCpuTime, total_memory_kib = (
+                ResourceMonitor.get_total_cpu_time_and_memory_usage()
+            )
+            job_time = time.time() - startTime
+            job_cpu_time = totalCpuTime - startClock
+            allocated_cpu_time = job_time * self.cores
+
+            if job_cpu_time > allocated_cpu_time and allocated_cpu_time > 0:
+                # Too much CPU was used by this job! Maybe we're using a batch
+                # system that doesn't/can't sandbox us and we started too many
+                # threads. Complain to the user!
+                excess_factor = job_cpu_time / allocated_cpu_time
+                fileStore.log_to_leader(
+                    f"Job {self.description} used {excess_factor:.2f}x more "
+                    f"CPU than the requested {self.cores} cores. Consider "
+                    f"increasing the job's required CPU cores or limiting the "
+                    f"number of processes/threads launched.",
+                    level=logging.WARNING
+                )
+
             # Finish up the stats
             if stats is not None:
-                totalCpuTime, total_memory_kib = (
-                    ResourceMonitor.get_total_cpu_time_and_memory_usage()
-                )
                 stats.jobs.append(
                     # TODO: We represent everything as strings in the stats
                     # even though the JSON transport can take bools and floats.
                     Expando(
                         start=str(startTime),
-                        time=str(time.time() - startTime),
-                        clock=str(totalCpuTime - startClock),
+                        time=str(job_time),
+                        clock=str(job_cpu_time),
                         class_name=self._jobName(),
                         memory=str(total_memory_kib),
                         requested_cores=str(self.cores), # TODO: Isn't this really consumed cores?