toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/cwl/utils.py

@@ -18,27 +18,17 @@ import logging
 import os
 import posixpath
 import stat
-from collections.abc import Iterable, MutableMapping, MutableSequence
+from collections.abc import Callable, Iterable, MutableMapping, MutableSequence
 from pathlib import PurePosixPath
-from typing import (
-    Any,
-    Callable,
-    TypeVar,
-    Union,
-    Optional,
-    cast,
-    MutableSequence,
-    MutableMapping,
-    TYPE_CHECKING,
-)
+from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
 from urllib.parse import unquote, urlparse
 
 if TYPE_CHECKING:
     # This module needs to be importable even if cwltool is not installed.
     from cwltool.utils import CWLObjectType, CWLOutputType
+
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
-from toil.jobStores.abstractJobStore import AbstractJobStore
 from toil.lib.url import URLAccess
 
 logger = logging.getLogger(__name__)
@@ -59,9 +49,9 @@ class CWLUnsupportedException(Exception):
 try:
     import cwltool.errors
 
-    CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION: Union[
-        type[cwltool.errors.UnsupportedRequirement], type[CWLUnsupportedException]
-    ] = cwltool.errors.UnsupportedRequirement
+    CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION: (
+        type[cwltool.errors.UnsupportedRequirement] | type[CWLUnsupportedException]
+    ) = cwltool.errors.UnsupportedRequirement
 except ImportError:
     CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION = CWLUnsupportedException
 
@@ -140,7 +130,7 @@ DirectoryStructure = dict[str, Union[str, "DirectoryStructure"]]
 
 def get_from_structure(
     dir_dict: DirectoryStructure, path: str
-) -> Union[str, DirectoryStructure, None]:
+) -> str | DirectoryStructure | None:
     """
     Given a relative path, follow it in the given directory structure.
 
@@ -154,7 +144,7 @@ def get_from_structure(
         return dir_dict
     if parts[0] in ("..", "/"):
         raise RuntimeError(f"Path {path} not resolvable in virtual directory")
-    found: Union[str, DirectoryStructure] = dir_dict
+    found: str | DirectoryStructure = dir_dict
     for part in parts:
         # Go down by each path component in turn
         if isinstance(found, str):
@@ -223,9 +213,7 @@ def download_structure(
                 )
             else:
                 # We need to download from some other kind of URL.
-                size, executable = URLAccess.read_from_url(
-                    value, open(dest_path, "wb")
-                )
+                size, executable = URLAccess.read_from_url(value, open(dest_path, "wb"))
                 if executable:
                     # Make the written file executable
                     os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)
@@ -243,27 +231,34 @@ def trim_mounts_op_down(file_or_directory: "CWLObjectType") -> None:
     return
 
 
-def sniff_location(file_or_directory: "CWLObjectType") -> Optional[str]:
+def sniff_location(file_or_directory: "CWLObjectType") -> str | None:
     """
     Get the local bare path for a CWL file or directory, or None.
 
     :return: None if we don't have a local path or file URI
     """
-    if file_or_directory.get('location') is None and file_or_directory.get('path') is None:
+    if (
+        file_or_directory.get("location") is None
+        and file_or_directory.get("path") is None
+    ):
         # file or directory is defined by contents or listing respectively, this is not redundant
         return None
     # Since we only consider mountable paths, if path is not file URI or bare path, don't consider it
-    path_or_url = cast(str, file_or_directory.get('location') or file_or_directory.get('path'))
+    path_or_url = cast(
+        str, file_or_directory.get("location") or file_or_directory.get("path")
+    )
     parsed = urlparse(path_or_url)
-    if parsed.scheme == 'file':
+    if parsed.scheme == "file":
         return unquote(parsed.path)
-    elif parsed.scheme == '':
+    elif parsed.scheme == "":
         return path_or_url
     else:
         return None
 
 
-def trim_mounts_op_up(file_or_directory: "CWLObjectType", op_down_ret: None, child_results: list[bool]) -> bool:
+def trim_mounts_op_up(
+    file_or_directory: "CWLObjectType", op_down_ret: None, child_results: list[bool]
+) -> bool:
     """
     Remove subtrees of the CWL file or directory object tree that only have redundant stuff in them.
 
@@ -282,40 +277,54 @@ def trim_mounts_op_up(file_or_directory: "CWLObjectType", op_down_ret: None, chi
     if own_path is None:
         return True
     # basename should be set as we are the implementation
-    own_basename = cast(str, file_or_directory['basename'])
+    own_basename = cast(str, file_or_directory["basename"])
 
     # If the basename does not match the path, then this is nonredundant
     if not own_path.endswith("/" + own_basename):
         return True
 
-    if file_or_directory['class'] == 'File':
+    if file_or_directory["class"] == "File":
         if any(child_results):
             # one of the children was detected as not redundant
             return True
-        for secondary in cast(MutableSequence[MutableMapping[str, "CWLOutputType"]], file_or_directory.get('secondaryFiles', [])):
+        for secondary in cast(
+            MutableSequence[MutableMapping[str, "CWLOutputType"]],
+            file_or_directory.get("secondaryFiles", []),
+        ):
             # secondary files should already be flagged nonredundant if they don't have either a path or location
             secondary_path = sniff_location(secondary)
-            secondary_basename = cast(str, secondary['basename'])
+            secondary_basename = cast(str, secondary["basename"])
             # If we swap the secondary basename for the primary basename in the primary path, and they don't match, then they are nonredundant
-            if os.path.join(own_path[:-len(own_basename)], secondary_basename) != secondary_path:
+            if (
+                os.path.join(own_path[: -len(own_basename)], secondary_basename)
+                != secondary_path
+            ):
                 return True
     else:
-        listings = cast(MutableSequence[MutableMapping[str, "CWLOutputType"]], file_or_directory.get('listing', []))
+        listings = cast(
+            MutableSequence[MutableMapping[str, "CWLOutputType"]],
+            file_or_directory.get("listing", []),
+        )
         if len(listings) == 0:
             return False
         # We assume child_results is in the same order as the directory listing
         # iterate backwards to avoid iteration issues
         for i in range(len(listings) - 1, -1, -1):
             if child_results[i] is False:
-                if os.path.join(own_path, cast(str, listings[i]['basename'])) == sniff_location(listings[i]):
+                if os.path.join(
+                    own_path, cast(str, listings[i]["basename"])
+                ) == sniff_location(listings[i]):
                     del listings[i]
         # If one of the listings was nonredundant, then this directory is also nonredundant
         if any(child_results):
             return True
     return False
 
+
 def remove_redundant_mounts(cwljob: "CWLObjectType") -> None:
     """
     Remove any redundant mount points from the listing. Modifies the CWL object in place.
     """
-    visit_cwl_class_and_reduce(cwljob, ["Directory", "File"], trim_mounts_op_down, trim_mounts_op_up)
+    visit_cwl_class_and_reduce(
+        cwljob, ["Directory", "File"], trim_mounts_op_down, trim_mounts_op_up
+    )

toil/fileStores/__init__.py

@@ -28,7 +28,9 @@ class FileID(str):
     the job store if unavailable in the ID.
     """
 
-    def __new__(cls, fileStoreID: str, *args: Any, **kwargs: dict[str, Any]) -> "FileID":
+    def __new__(
+        cls, fileStoreID: str, *args: Any, **kwargs: dict[str, Any]
+    ) -> "FileID":
         return super().__new__(cls, fileStoreID)
 
     def __init__(self, fileStoreID: str, size: int, executable: bool = False) -> None:

toil/fileStores/abstractFileStore.py

@@ -14,7 +14,7 @@
 import logging
 import os
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Iterator
+from collections.abc import Callable, Generator, Iterator
 from contextlib import contextmanager
 from tempfile import mkstemp
 from threading import Event, Semaphore
@@ -22,10 +22,8 @@ from typing import (
     IO,
     TYPE_CHECKING,
     Any,
-    Callable,
     ContextManager,
     Literal,
-    Optional,
     Union,
     cast,
     overload,
@@ -118,7 +116,7 @@ class AbstractFileStore(ABC):
         )
         self.jobName: str = str(self.jobDesc)
         self.waitForPreviousCommit = waitForPreviousCommit
-        self.logging_messages: list[dict[str, Union[int, str]]] = []
+        self.logging_messages: list[dict[str, int | str]] = []
         self.logging_user_streams: list[dict[str, str]] = []
         # Records file IDs of files deleted during the current job. Doesn't get
         # committed back until the job is completely successful, because if the
@@ -129,7 +127,7 @@ class AbstractFileStore(ABC):
         # the accessed files of failed jobs.
         self._accessLog: list[tuple[str, ...]] = []
         # Holds total bytes of observed disk usage for the last job run under open()
-        self._job_disk_used: Optional[int] = None
+        self._job_disk_used: int | None = None
 
     @staticmethod
     def createFileStore(
@@ -137,14 +135,14 @@ class AbstractFileStore(ABC):
         jobDesc: JobDescription,
         file_store_dir: str,
         waitForPreviousCommit: Callable[[], Any],
-        caching: Optional[bool],
+        caching: bool | None,
     ) -> Union["NonCachingFileStore", "CachingFileStore"]:
         """Create a concreate FileStore."""
         # Defer these imports until runtime, since these classes depend on us
         from toil.fileStores.cachingFileStore import CachingFileStore
         from toil.fileStores.nonCachingFileStore import NonCachingFileStore
 
-        fileStoreCls: Union[type["CachingFileStore"], type["NonCachingFileStore"]] = (
+        fileStoreCls: type["CachingFileStore"] | type["NonCachingFileStore"] = (
             CachingFileStore if caching else NonCachingFileStore
         )
         return fileStoreCls(jobStore, jobDesc, file_store_dir, waitForPreviousCommit)
@@ -152,8 +150,8 @@ class AbstractFileStore(ABC):
     @staticmethod
     def shutdownFileStore(
         workflowID: str,
-        config_work_dir: Optional[str],
-        config_coordination_dir: Optional[str],
+        config_work_dir: str | None,
+        config_coordination_dir: str | None,
     ) -> None:
         """
         Carry out any necessary filestore-specific cleanup.
@@ -230,7 +228,7 @@ class AbstractFileStore(ABC):
             else:
                 self.log_to_leader(disk_usage, level=logging.DEBUG)
 
-    def get_disk_usage(self) -> Optional[int]:
+    def get_disk_usage(self) -> int | None:
         """
         Get the number of bytes of disk used by the last job run under open().
 
@@ -254,7 +252,7 @@ class AbstractFileStore(ABC):
         return os.path.abspath(mkdtemp(dir=self.localTempDir))
 
     def getLocalTempFile(
-        self, suffix: Optional[str] = None, prefix: Optional[str] = None
+        self, suffix: str | None = None, prefix: str | None = None
     ) -> str:
         """
         Get a new local temporary file that will persist for the duration of the job.
@@ -278,7 +276,7 @@ class AbstractFileStore(ABC):
         return os.path.abspath(tmpFile)
 
     def getLocalTempFileName(
-        self, suffix: Optional[str] = None, prefix: Optional[str] = None
+        self, suffix: str | None = None, prefix: str | None = None
     ) -> str:
         """
         Get a valid name for a new local file. Don't actually create a file at the path.
@@ -330,9 +328,9 @@ class AbstractFileStore(ABC):
     def writeGlobalFileStream(
         self,
         cleanup: bool = False,
-        basename: Optional[str] = None,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
+        basename: str | None = None,
+        encoding: str | None = None,
+        errors: str | None = None,
     ) -> Iterator[tuple[WriteWatchingStream, FileID]]:
         """
         Similar to writeGlobalFile, but allows the writing of a stream to the job store.
@@ -423,7 +421,7 @@ class AbstractFileStore(ABC):
                     logger.log(log_level, "Streamed file '%s'", *item)
 
     def logAccess(
-        self, fileStoreID: Union[FileID, str], destination: Union[str, None] = None
+        self, fileStoreID: FileID | str, destination: str | None = None
     ) -> None:
         """
         Record that the given file was read by the job.
@@ -445,7 +443,7 @@ class AbstractFileStore(ABC):
     def readGlobalFile(
         self,
         fileStoreID: str,
-        userPath: Optional[str] = None,
+        userPath: str | None = None,
         cache: bool = True,
         mutable: bool = False,
         symlink: bool = False,
@@ -486,21 +484,21 @@ class AbstractFileStore(ABC):
         self,
         fileStoreID: str,
         encoding: Literal[None] = None,
-        errors: Optional[str] = None,
+        errors: str | None = None,
     ) -> ContextManager[IO[bytes]]: ...
 
     @overload
     def readGlobalFileStream(
-        self, fileStoreID: str, encoding: str, errors: Optional[str] = None
+        self, fileStoreID: str, encoding: str, errors: str | None = None
     ) -> ContextManager[IO[str]]: ...
 
     @abstractmethod
     def readGlobalFileStream(
         self,
         fileStoreID: str,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-    ) -> ContextManager[Union[IO[bytes], IO[str]]]:
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> ContextManager[IO[bytes] | IO[str]]:
         """
         Read a stream from the job store; similar to readGlobalFile.
 
@@ -519,7 +517,7 @@ class AbstractFileStore(ABC):
         """
         raise NotImplementedError()
 
-    def getGlobalFileSize(self, fileStoreID: Union[FileID, str]) -> int:
+    def getGlobalFileSize(self, fileStoreID: FileID | str) -> int:
         """
         Get the size of the file pointed to by the given ID, in bytes.
 
@@ -547,7 +545,7 @@ class AbstractFileStore(ABC):
         return cast(int, size)
 
     @abstractmethod
-    def deleteLocalFile(self, fileStoreID: Union[FileID, str]) -> None:
+    def deleteLocalFile(self, fileStoreID: FileID | str) -> None:
         """
         Delete local copies of files associated with the provided job store ID.
 
@@ -565,7 +563,7 @@ class AbstractFileStore(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    def deleteGlobalFile(self, fileStoreID: Union[FileID, str]) -> None:
+    def deleteGlobalFile(self, fileStoreID: FileID | str) -> None:
         """
         Delete local files and then permanently deletes them from the job store.
 
@@ -580,13 +578,13 @@ class AbstractFileStore(ABC):
     # and the job store.
     @deprecated(new_function_name="import_file")
     def importFile(
-        self, srcUrl: str, sharedFileName: Optional[str] = None
-    ) -> Optional[FileID]:
+        self, srcUrl: str, sharedFileName: str | None = None
+    ) -> FileID | None:
         return self.import_file(srcUrl, sharedFileName)
 
     def import_file(
-        self, src_uri: str, shared_file_name: Optional[str] = None
-    ) -> Optional[FileID]:
+        self, src_uri: str, shared_file_name: str | None = None
+    ) -> FileID | None:
         return self.jobStore.import_file(src_uri, shared_file_name=shared_file_name)
 
     @deprecated(new_function_name="export_file")
@@ -625,7 +623,7 @@ class AbstractFileStore(ABC):
         @classmethod
         @abstractmethod
         @contextmanager
-        def open(cls, outer: Optional[Any] = None) -> Iterator[Any]:
+        def open(cls, outer: Any | None = None) -> Iterator[Any]:
             """
             This is a context manager that state file and reads it into an object that is returned
             to the user in the yield.

toil/fileStores/cachingFileStore.py

@@ -22,10 +22,10 @@ import sqlite3
 import stat
 import threading
 import time
-from collections.abc import Generator, Iterator, Sequence
+from collections.abc import Callable, Generator, Iterator, Sequence
 from contextlib import contextmanager
 from tempfile import mkstemp
-from typing import Any, Callable, Optional
+from typing import Any
 
 from toil.common import cacheDirName, getFileSystemSize
 from toil.fileStores import FileID
@@ -221,7 +221,7 @@ class CachingFileStore(AbstractFileStore):
         logger.debug("Starting job (%s) with ID (%s).", self.jobName, self.jobID)
 
         # When the job actually starts, we will fill this in with the job's disk requirement.
-        self.jobDiskBytes: Optional[float] = None
+        self.jobDiskBytes: float | None = None
 
         # We need to track what attempt of the workflow we are, to prevent crosstalk between attempts' caches.
         self.workflowAttemptNumber = self.jobStore.config.workflowAttemptNumber
@@ -386,7 +386,7 @@ class CachingFileStore(AbstractFileStore):
         ],
     )
     def _static_read(
-        cur: sqlite3.Cursor, query: str, args: Optional[Sequence[Any]] = ()
+        cur: sqlite3.Cursor, query: str, args: Sequence[Any] | None = ()
     ) -> Iterator[Any]:
         """
         Read from the database.
@@ -398,7 +398,7 @@ class CachingFileStore(AbstractFileStore):
         # All the real work is the decorators
         return cur.execute(query, args)
 
-    def _read(self, query: str, args: Optional[Sequence[Any]] = ()) -> Iterator[Any]:
+    def _read(self, query: str, args: Sequence[Any] | None = ()) -> Iterator[Any]:
         """
         Read from the database using the instance's connection.
 
@@ -2010,7 +2010,7 @@ class CachingFileStore(AbstractFileStore):
         self,
         file_store_id: FileID,
         reader_id: str,
-        local_file_path: Optional[str] = None,
+        local_file_path: str | None = None,
     ) -> Generator:
         """
         Get a context manager that gives you either the local file path for a
@@ -2275,7 +2275,7 @@ class CachingFileStore(AbstractFileStore):
         if len(self.jobDesc.filesToDelete) > 0:
             raise RuntimeError("Job is already in the process of being committed!")
 
-        state_to_commit: Optional[JobDescription] = None
+        state_to_commit: JobDescription | None = None
 
         if jobState:
             # Clone the current job description, so that further updates to it
@@ -2308,7 +2308,7 @@ class CachingFileStore(AbstractFileStore):
         )
         self.commitThread.start()
 
-    def startCommitThread(self, state_to_commit: Optional[JobDescription]):
+    def startCommitThread(self, state_to_commit: JobDescription | None):
         """
         Run in a thread to actually commit the current job.
         """

toil/fileStores/nonCachingFileStore.py

@@ -16,20 +16,9 @@ import logging
 import os
 import tempfile
 from collections import defaultdict
-from collections.abc import Generator, Iterator
+from collections.abc import Callable, Generator, Iterator
 from contextlib import contextmanager
-from typing import (
-    IO,
-    Any,
-    Callable,
-    ContextManager,
-    DefaultDict,
-    Literal,
-    Optional,
-    Union,
-    cast,
-    overload,
-)
+from typing import IO, Any, ContextManager, DefaultDict, Literal, cast, overload
 
 import dill
 
@@ -61,13 +50,13 @@ class NonCachingFileStore(AbstractFileStore):
     ) -> None:
         super().__init__(jobStore, jobDesc, file_store_dir, waitForPreviousCommit)
         # This will be defined in the `open` method.
-        self.jobStateFile: Optional[str] = None
+        self.jobStateFile: str | None = None
         self.localFileMap: DefaultDict[str, list[str]] = defaultdict(list)
 
         self.check_for_state_corruption()
 
     @staticmethod
-    def check_for_coordination_corruption(coordination_dir: Optional[str]) -> None:
+    def check_for_coordination_corruption(coordination_dir: str | None) -> None:
         """
         Make sure the coordination directory hasn't been deleted unexpectedly.
 
@@ -145,7 +134,7 @@ class NonCachingFileStore(AbstractFileStore):
     def readGlobalFile(
         self,
         fileStoreID: str,
-        userPath: Optional[str] = None,
+        userPath: str | None = None,
         cache: bool = True,
         mutable: bool = False,
         symlink: bool = False,
@@ -169,12 +158,12 @@ class NonCachingFileStore(AbstractFileStore):
         self,
         fileStoreID: str,
         encoding: Literal[None] = None,
-        errors: Optional[str] = None,
+        errors: str | None = None,
     ) -> ContextManager[IO[bytes]]: ...
 
     @overload
     def readGlobalFileStream(
-        self, fileStoreID: str, encoding: str, errors: Optional[str] = None
+        self, fileStoreID: str, encoding: str, errors: str | None = None
     ) -> ContextManager[IO[str]]: ...
 
     # TODO: This seems to hit https://github.com/python/mypy/issues/11373
@@ -184,9 +173,9 @@ class NonCachingFileStore(AbstractFileStore):
     def readGlobalFileStream(
         self,
         fileStoreID: str,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-    ) -> Iterator[Union[IO[bytes], IO[str]]]:
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> Iterator[IO[bytes] | IO[str]]:
         with self.jobStore.read_file_stream(
             fileStoreID, encoding=encoding, errors=errors
         ) as f: