toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/cwl/utils.py

@@ -16,6 +16,9 @@
 
 import logging
 import os
+from pathlib import PurePosixPath
+import posixpath
+import stat
 from typing import (
     Any,
     Callable,
@@ -24,7 +27,6 @@ from typing import (
     List,
     MutableMapping,
     MutableSequence,
-    Tuple,
     Type,
     TypeVar,
     Union,
@@ -32,6 +34,7 @@ from typing import (
 
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
+from toil.jobStores.abstractJobStore import AbstractJobStore
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +132,32 @@ def visit_cwl_class_and_reduce(
 
 DirectoryStructure = Dict[str, Union[str, "DirectoryStructure"]]
 
+def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, DirectoryStructure, None]:
+    """
+    Given a relative path, follow it in the given directory structure.
+
+    Return the string URI for files, the directory dict for
+    subdirectories, or None for nonexistent things.
+    """
+
+    # Resolve .. and split into path components
+    parts = PurePosixPath(posixpath.normpath(path)).parts
+    if len(parts) == 0:
+        return dir_dict
+    if parts[0] in ('..', '/'):
+        raise RuntimeError(f"Path {path} not resolvable in virtual directory")
+    found: Union[str, DirectoryStructure] = dir_dict
+    for part in parts:
+        # Go down by each path component in turn
+        if isinstance(found, str):
+            # Looking for a subdirectory of a file, which doesn't exist
+            return None
+        if part not in found:
+            return None
+        found = found[part]
+    # Now we're at the place we want to be.
+    return found
+
 
 def download_structure(
     file_store: AbstractFileStore,
@@ -140,17 +169,21 @@ def download_structure(
     """
     Download nested dictionary from the Toil file store to a local path.
 
+    Guaranteed to fill the structure with real files, and not symlinks out of
+    it to elsewhere. File URIs may be toilfile: URIs or any other URI that
+    Toil's job store system can read.
+
     :param file_store: The Toil file store to download from.
 
-    :param index: Maps from downloaded file path back to input Toil URI.
+    :param index: Maps from downloaded file path back to input URI.
 
     :param existing: Maps from file_store_id URI to downloaded file path.
 
     :param dir_dict: a dict from string to string (for files) or dict (for
-    subdirectories) describing a directory structure.
+        subdirectories) describing a directory structure.
 
     :param into_dir: The directory to download the top-level dict's files
-    into.
+        into.
     """
     logger.debug("Downloading directory with %s items", len(dir_dict))
 
@@ -167,14 +200,26 @@ def download_structure(
             download_structure(file_store, index, existing, value, subdir)
         else:
             # This must be a file path uploaded to Toil.
-            assert isinstance(value, str)
-            assert value.startswith("toilfile:")
+            if not isinstance(value, str):
+                raise RuntimeError(f"Did not find a file at {value}.")
+
             logger.debug("Downloading contained file '%s'", name)
             dest_path = os.path.join(into_dir, name)
-            # So download the file into place
-            file_store.readGlobalFile(
-                FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=True
-            )
+
+            if value.startswith("toilfile:"):
+                # So download the file into place.
+                # Make sure to get a real copy of the file because we may need to
+                # mount the directory into a container as a whole.
+                file_store.readGlobalFile(
+                    FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=False
+                )
+            else:
+                # We need to download from some other kind of URL.
+                size, executable = AbstractJobStore.read_from_url(value, open(dest_path, 'wb'))
+                if executable:
+                    # Make the written file executable
+                    os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)
+
             # Update the index dicts
             # TODO: why?
             index[dest_path] = value

toil/exceptions.py

@@ -36,7 +36,7 @@ class FailedJobsException(Exception):
             for job_desc in failed_jobs:
                 if job_desc.logJobStoreFileID:
                     with job_desc.getLogFileHandle(job_store) as f:
-                        self.msg += "\n" + StatsAndLogging.formatLogStream(f, job_desc)
+                        self.msg += "\n" + StatsAndLogging.formatLogStream(f, f'Log from job "{job_desc}"')
         # catch failures to prepare more complex details and only return the basics
         except Exception:
             logger.exception("Exception when compiling information about failed jobs")

toil/fileStores/__init__.py

@@ -40,7 +40,7 @@ class FileID(str):
 
     def pack(self) -> str:
         """Pack the FileID into a string so it can be passed through external code."""
-        return f'{self.size}:{int(self.executable)}:{self}'
+        return f'{self.size}:{"1" if self.executable else "0"}:{self}'
 
     @classmethod
     def forPath(cls, fileStoreID: str, filePath: str) -> 'FileID':
@@ -54,7 +54,7 @@ class FileID(str):
         vals = packedFileStoreID.split(':', 2)
         # Break up the packed value
         size = int(vals[0])
-        executable = bool(vals[1])
+        executable = (vals[1] == "1")
         value = vals[2]
         # Create the FileID
         return cls(value, size, executable)

toil/fileStores/abstractFileStore.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 import logging
 import os
-import tempfile
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
+from tempfile import mkstemp
 from threading import Event, Semaphore
 from typing import (IO,
                     TYPE_CHECKING,
@@ -26,21 +26,24 @@ from typing import (IO,
                     Generator,
                     Iterator,
                     List,
+                    Literal,
                     Optional,
                     Set,
                     Tuple,
                     Type,
                     Union,
-                    cast)
+                    cast,
+                    overload)
 
 import dill
 
-from toil.common import Toil, cacheDirName
+from toil.common import Toil, cacheDirName, getDirSizeRecursively
 from toil.fileStores import FileID
 from toil.job import Job, JobDescription
 from toil.jobStores.abstractJobStore import AbstractJobStore
 from toil.lib.compatibility import deprecated
-from toil.lib.io import WriteWatchingStream
+from toil.lib.conversions import bytes2human
+from toil.lib.io import WriteWatchingStream, mkdtemp
 
 logger = logging.getLogger(__name__)
 
@@ -114,19 +117,18 @@ class AbstractFileStore(ABC):
             self.jobDesc.command.split()[1] if self.jobDesc.command else ""
         )
         self.waitForPreviousCommit = waitForPreviousCommit
-        self.loggingMessages: List[Dict[str, Union[int, str]]] = []
+        self.logging_messages: List[Dict[str, Union[int, str]]] = []
+        self.logging_user_streams: List[dict[str, str]] = []
         # Records file IDs of files deleted during the current job. Doesn't get
         # committed back until the job is completely successful, because if the
         # job is re-run it will need to be able to re-delete these files.
         # This is a set of str objects, not FileIDs.
         self.filesToDelete: Set[str] = set()
-        # Records IDs of jobs that need to be deleted when the currently
-        # running job is cleaned up.
-        # May be modified by the worker to actually delete jobs!
-        self.jobsToDelete: Set[str] = set()
         # Holds records of file ID, or file ID and local path, for reporting
         # the accessed files of failed jobs.
         self._accessLog: List[Tuple[str, ...]] = []
+        # Holds total bytes of observed disk usage for the last job run under open()
+        self._job_disk_used: Optional[int] = None
 
     @staticmethod
     def createFileStore(
@@ -190,6 +192,7 @@ class AbstractFileStore(ABC):
         :param job: The job instance of the toil job to run.
         """
         failed = True
+        job_requested_disk = job.disk
         try:
             yield
             failed = False
@@ -199,6 +202,33 @@ class AbstractFileStore(ABC):
             if failed:
                 self._dumpAccessLogs()
 
+            # See how much disk space is used at the end of the job.
+            # Not a real peak disk usage, but close enough to be useful for warning the user.
+            self._job_disk_used = getDirSizeRecursively(self.localTempDir)
+
+            # Report disk usage
+            percent: float = 0.0
+            if job_requested_disk and job_requested_disk > 0:
+                percent = float(self._job_disk_used) / job_requested_disk * 100
+            disk_usage: str = (f"Job {self.jobName} used {percent:.2f}% disk ({bytes2human(self._job_disk_used)}B [{self._job_disk_used}B] used, "
+                               f"{bytes2human(job_requested_disk)}B [{job_requested_disk}B] requested).")
+            if self._job_disk_used > job_requested_disk:
+                self.log_to_leader("Job used more disk than requested. For CWL, consider increasing the outdirMin "
+                                 f"requirement, otherwise, consider increasing the disk requirement. {disk_usage}",
+                                 level=logging.WARNING)
+            else:
+                self.log_to_leader(disk_usage, level=logging.DEBUG)
+
+    def get_disk_usage(self) -> Optional[int]:
+        """
+        Get the number of bytes of disk used by the last job run under open().
+
+        Disk usage is measured at the end of the job.
+        TODO: Sample periodically and record peak usage.
+        """
+        return self._job_disk_used
+
+
     # Functions related to temp files and directories
     def getLocalTempDir(self) -> str:
         """
@@ -211,7 +241,7 @@ class AbstractFileStore(ABC):
                  to be deleted once the job terminates, removing all files it
                  contains recursively.
         """
-        return os.path.abspath(tempfile.mkdtemp(dir=self.localTempDir))
+        return os.path.abspath(mkdtemp(dir=self.localTempDir))
 
     def getLocalTempFile(self, suffix: Optional[str] = None, prefix: Optional[str] = None) -> str:
         """
@@ -227,7 +257,7 @@ class AbstractFileStore(ABC):
                  for the duration of the job only, and is guaranteed to be deleted
                  once the job terminates.
         """
-        handle, tmpFile = tempfile.mkstemp(
+        handle, tmpFile = mkstemp(
             suffix=".tmp" if suffix is None else suffix,
             prefix="tmp" if prefix is None else prefix,
             dir=self.localTempDir
@@ -417,6 +447,21 @@ class AbstractFileStore(ABC):
         """
         raise NotImplementedError()
 
+    @overload
+    def readGlobalFileStream(
+        self,
+        fileStoreID: str,
+        encoding: Literal[None] = None,
+        errors: Optional[str] = None,
+    ) -> ContextManager[IO[bytes]]:
+        ...
+
+    @overload
+    def readGlobalFileStream(
+        self, fileStoreID: str, encoding: str, errors: Optional[str] = None
+    ) -> ContextManager[IO[str]]:
+        ...
+
     @abstractmethod
     def readGlobalFileStream(
         self,
@@ -589,7 +634,7 @@ class AbstractFileStore(ABC):
             os.rename(fileName + '.tmp', fileName)
 
     # Functions related to logging
-    def logToMaster(self, text: str, level: int = logging.INFO) -> None:
+    def log_to_leader(self, text: str, level: int = logging.INFO) -> None:
         """
         Send a logging message to the leader. The message will also be \
         logged by the worker at the same level.
@@ -598,7 +643,30 @@ class AbstractFileStore(ABC):
         :param level: The logging level.
         """
         logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
-        self.loggingMessages.append(dict(text=text, level=level))
+        self.logging_messages.append(dict(text=text, level=level))
+
+
+    @deprecated(new_function_name='export_file')
+    def logToMaster(self, text: str, level: int = logging.INFO) -> None:
+        self.log_to_leader(text, level)
+
+    def log_user_stream(self, name: str, stream: IO[bytes]) -> None:
+        """
+        Send a stream of UTF-8 text to the leader as a named log stream.
+
+        Useful for things like the error logs of Docker containers. The leader
+        will show it to the user or organize it appropriately for user-level
+        log information.
+
+        :param name: A hierarchical, .-delimited string.
+        :param stream: A stream of encoded text. Encoding errors will be
+            tolerated.
+        """
+
+        # Read the whole stream into memory
+        steam_data = stream.read().decode('utf-8', errors='replace')
+        # And remember it for the worker to fish out
+        self.logging_user_streams.append(dict(name=name, text=steam_data))
 
     # Functions run after the completion of the job.
     @abstractmethod
@@ -606,7 +674,13 @@ class AbstractFileStore(ABC):
         """
         Update the status of the job on the disk.
 
-        May start an asynchronous process. Call waitForCommit() to wait on that process.
+        May bump the version number of the job.
+
+        May start an asynchronous process. Call waitForCommit() to wait on that
+        process. You must waitForCommit() before committing any further updates
+        to the job. During the asynchronous process, it is safe to modify the
+        job; modifications after this call will not be committed until the next
+        call.
 
         :param jobState: If True, commit the state of the FileStore's job,
                     and file deletes. Otherwise, commit only file creates/updates.