toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/cwl/utils.py

@@ -16,6 +16,9 @@
 
 import logging
 import os
+from pathlib import PurePosixPath
+import posixpath
+import stat
 from typing import (
     Any,
     Callable,
@@ -24,7 +27,6 @@ from typing import (
     List,
     MutableMapping,
     MutableSequence,
-    Tuple,
     Type,
     TypeVar,
     Union,
@@ -32,6 +34,7 @@ from typing import (
 
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
+from toil.jobStores.abstractJobStore import AbstractJobStore
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +132,32 @@ def visit_cwl_class_and_reduce(
 
 DirectoryStructure = Dict[str, Union[str, "DirectoryStructure"]]
 
+def get_from_structure(dir_dict: DirectoryStructure, path: str) -> Union[str, DirectoryStructure, None]:
+    """
+    Given a relative path, follow it in the given directory structure.
+
+    Return the string URI for files, the directory dict for
+    subdirectories, or None for nonexistent things.
+    """
+
+    # Resolve .. and split into path components
+    parts = PurePosixPath(posixpath.normpath(path)).parts
+    if len(parts) == 0:
+        return dir_dict
+    if parts[0] in ('..', '/'):
+        raise RuntimeError(f"Path {path} not resolvable in virtual directory")
+    found: Union[str, DirectoryStructure] = dir_dict
+    for part in parts:
+        # Go down by each path component in turn
+        if isinstance(found, str):
+            # Looking for a subdirectory of a file, which doesn't exist
+            return None
+        if part not in found:
+            return None
+        found = found[part]
+    # Now we're at the place we want to be.
+    return found
+
 
 def download_structure(
     file_store: AbstractFileStore,
@@ -140,17 +169,21 @@ def download_structure(
     """
     Download nested dictionary from the Toil file store to a local path.
 
+    Guaranteed to fill the structure with real files, and not symlinks out of
+    it to elsewhere. File URIs may be toilfile: URIs or any other URI that
+    Toil's job store system can read.
+
     :param file_store: The Toil file store to download from.
 
-    :param index: Maps from downloaded file path back to input Toil URI.
+    :param index: Maps from downloaded file path back to input URI.
 
     :param existing: Maps from file_store_id URI to downloaded file path.
 
     :param dir_dict: a dict from string to string (for files) or dict (for
-    subdirectories) describing a directory structure.
+        subdirectories) describing a directory structure.
 
     :param into_dir: The directory to download the top-level dict's files
-    into.
+        into.
     """
     logger.debug("Downloading directory with %s items", len(dir_dict))
 
@@ -167,14 +200,26 @@ def download_structure(
             download_structure(file_store, index, existing, value, subdir)
         else:
             # This must be a file path uploaded to Toil.
-            assert isinstance(value, str)
-            assert value.startswith("toilfile:")
+            if not isinstance(value, str):
+                raise RuntimeError(f"Did not find a file at {value}.")
+
             logger.debug("Downloading contained file '%s'", name)
             dest_path = os.path.join(into_dir, name)
-            # So download the file into place
-            file_store.readGlobalFile(
-                FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=True
-            )
+
+            if value.startswith("toilfile:"):
+                # So download the file into place.
+                # Make sure to get a real copy of the file because we may need to
+                # mount the directory into a container as a whole.
+                file_store.readGlobalFile(
+                    FileID.unpack(value[len("toilfile:") :]), dest_path, symlink=False
+                )
+            else:
+                # We need to download from some other kind of URL.
+                size, executable = AbstractJobStore.read_from_url(value, open(dest_path, 'wb'))
+                if executable:
+                    # Make the written file executable
+                    os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)
+
             # Update the index dicts
             # TODO: why?
             index[dest_path] = value

toil/fileStores/__init__.py

@@ -40,7 +40,7 @@ class FileID(str):
 
     def pack(self) -> str:
         """Pack the FileID into a string so it can be passed through external code."""
-        return f'{self.size}:{int(self.executable)}:{self}'
+        return f'{self.size}:{"1" if self.executable else "0"}:{self}'
 
     @classmethod
     def forPath(cls, fileStoreID: str, filePath: str) -> 'FileID':
@@ -54,7 +54,7 @@ class FileID(str):
         vals = packedFileStoreID.split(':', 2)
         # Break up the packed value
         size = int(vals[0])
-        executable = bool(vals[1])
+        executable = (vals[1] == "1")
         value = vals[2]
         # Create the FileID
         return cls(value, size, executable)

toil/fileStores/abstractFileStore.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 import logging
 import os
-import tempfile
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
+from tempfile import mkstemp
 from threading import Event, Semaphore
 from typing import (IO,
                     TYPE_CHECKING,
@@ -26,12 +26,14 @@ from typing import (IO,
                     Generator,
                     Iterator,
                     List,
+                    Literal,
                     Optional,
                     Set,
                     Tuple,
                     Type,
                     Union,
-                    cast)
+                    cast,
+                    overload)
 
 import dill
 
@@ -40,7 +42,7 @@ from toil.fileStores import FileID
 from toil.job import Job, JobDescription
 from toil.jobStores.abstractJobStore import AbstractJobStore
 from toil.lib.compatibility import deprecated
-from toil.lib.io import WriteWatchingStream
+from toil.lib.io import WriteWatchingStream, mkdtemp
 
 logger = logging.getLogger(__name__)
 
@@ -120,10 +122,6 @@ class AbstractFileStore(ABC):
         # job is re-run it will need to be able to re-delete these files.
         # This is a set of str objects, not FileIDs.
         self.filesToDelete: Set[str] = set()
-        # Records IDs of jobs that need to be deleted when the currently
-        # running job is cleaned up.
-        # May be modified by the worker to actually delete jobs!
-        self.jobsToDelete: Set[str] = set()
         # Holds records of file ID, or file ID and local path, for reporting
         # the accessed files of failed jobs.
         self._accessLog: List[Tuple[str, ...]] = []
@@ -211,7 +209,7 @@ class AbstractFileStore(ABC):
                  to be deleted once the job terminates, removing all files it
                  contains recursively.
         """
-        return os.path.abspath(tempfile.mkdtemp(dir=self.localTempDir))
+        return os.path.abspath(mkdtemp(dir=self.localTempDir))
 
     def getLocalTempFile(self, suffix: Optional[str] = None, prefix: Optional[str] = None) -> str:
         """
@@ -227,7 +225,7 @@ class AbstractFileStore(ABC):
                  for the duration of the job only, and is guaranteed to be deleted
                  once the job terminates.
         """
-        handle, tmpFile = tempfile.mkstemp(
+        handle, tmpFile = mkstemp(
             suffix=".tmp" if suffix is None else suffix,
             prefix="tmp" if prefix is None else prefix,
             dir=self.localTempDir
@@ -417,6 +415,21 @@ class AbstractFileStore(ABC):
         """
         raise NotImplementedError()
 
+    @overload
+    def readGlobalFileStream(
+        self,
+        fileStoreID: str,
+        encoding: Literal[None] = None,
+        errors: Optional[str] = None,
+    ) -> ContextManager[IO[bytes]]:
+        ...
+
+    @overload
+    def readGlobalFileStream(
+        self, fileStoreID: str, encoding: str, errors: Optional[str] = None
+    ) -> ContextManager[IO[str]]:
+        ...
+
     @abstractmethod
     def readGlobalFileStream(
         self,
@@ -589,7 +602,7 @@ class AbstractFileStore(ABC):
             os.rename(fileName + '.tmp', fileName)
 
     # Functions related to logging
-    def logToMaster(self, text: str, level: int = logging.INFO) -> None:
+    def log_to_leader(self, text: str, level: int = logging.INFO) -> None:
         """
         Send a logging message to the leader. The message will also be \
         logged by the worker at the same level.
@@ -600,13 +613,25 @@ class AbstractFileStore(ABC):
         logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
         self.loggingMessages.append(dict(text=text, level=level))
 
+
+    @deprecated(new_function_name='export_file')
+    def logToMaster(self, text: str, level: int = logging.INFO) -> None:
+        self.log_to_leader(text, level)
+        
+
     # Functions run after the completion of the job.
     @abstractmethod
     def startCommit(self, jobState: bool = False) -> None:
         """
         Update the status of the job on the disk.
 
-        May start an asynchronous process. Call waitForCommit() to wait on that process.
+        May bump the version number of the job.
+
+        May start an asynchronous process. Call waitForCommit() to wait on that
+        process. You must waitForCommit() before committing any further updates
+        to the job. During the asynchronous process, it is safe to modify the
+        job; modifications after this call will not be committed until the next
+        call.
 
         :param jobState: If True, commit the state of the FileStore's job,
                     and file deletes. Otherwise, commit only file creates/updates.