toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/jobStores/aws/utils.py

@@ -11,33 +11,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import base64
-import bz2
 import logging
 import os
-import types
 from ssl import SSLError
-from typing import TYPE_CHECKING, IO, Optional, cast, Any
+from typing import IO, TYPE_CHECKING, Any
 
 from boto3.s3.transfer import TransferConfig
 from botocore.client import Config
 from botocore.exceptions import ClientError
 
 from toil.lib.aws import AWSServerErrors, session
-from toil.lib.aws.utils import connection_error, get_bucket_region
+from toil.lib.aws.utils import get_bucket_region
 from toil.lib.compatibility import compat_bytes
-from toil.lib.retry import (
-    DEFAULT_DELAYS,
-    DEFAULT_TIMEOUT,
-    get_error_code,
-    get_error_message,
-    get_error_status,
-    old_retry,
-    retry,
-)
+from toil.lib.retry import get_error_code, get_error_message, retry
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3 import S3Client, S3ServiceResource
+    from mypy_boto3_s3 import S3ServiceResource
     from mypy_boto3_s3.type_defs import CopySourceTypeDef
 
 logger = logging.getLogger(__name__)
@@ -66,9 +55,9 @@ def uploadFromPath(
     resource: "S3ServiceResource",
     bucketName: str,
     fileID: str,
-    headerArgs: Optional[dict[str, Any]] = None,
+    headerArgs: dict[str, Any] | None = None,
     partSize: int = 50 << 20,
-) -> Optional[str]:
+) -> str | None:
     """
     Uploads a file to s3, using multipart uploading if applicable
 
@@ -112,9 +101,9 @@ def uploadFile(
     resource: "S3ServiceResource",
     bucketName: str,
     fileID: str,
-    headerArgs: Optional[dict[str, Any]] = None,
+    headerArgs: dict[str, Any] | None = None,
     partSize: int = 50 << 20,
-) -> Optional[str]:
+) -> str | None:
     """
     Upload a readable object to s3, using multipart uploading if applicable.
     :param readable: a readable stream or a file path to upload to s3
@@ -172,11 +161,11 @@ def copyKeyMultipart(
     srcKeyVersion: str,
     dstBucketName: str,
     dstKeyName: str,
-    sseAlgorithm: Optional[str] = None,
-    sseKey: Optional[str] = None,
-    copySourceSseAlgorithm: Optional[str] = None,
-    copySourceSseKey: Optional[str] = None,
-) -> Optional[str]:
+    sseAlgorithm: str | None = None,
+    sseKey: str | None = None,
+    copySourceSseAlgorithm: str | None = None,
+    copySourceSseKey: str | None = None,
+) -> str | None:
     """
     Copies a key from a source key to a destination key in multiple parts. Note that if the
     destination key exists it will be overwritten implicitly, and if it does not exist a new

toil/jobStores/fileJobStore.py

@@ -23,7 +23,7 @@ import time
 import uuid
 from collections.abc import Iterable, Iterator
 from contextlib import contextmanager
-from typing import IO, Literal, Optional, Union, overload
+from typing import IO, Literal, overload
 from urllib.parse import ParseResult, quote, unquote
 
 from toil.fileStores import FileID
@@ -327,7 +327,7 @@ class FileJobStore(AbstractJobStore, URLAccess):
         # linking is not done be default because of issue #1755
         # TODO: is hardlinking ever actually done?
         src_path = self._extract_path_from_url(src_path)
-        if self.linkImports and not hardlink and symlink:
+        if self.linkImports and not hardlink and symlink and src_path != "/dev/null":
             os.symlink(os.path.realpath(src_path), dst_path)
         else:
             atomic_copy(src_path, dst_path)
@@ -703,33 +703,33 @@ class FileJobStore(AbstractJobStore, URLAccess):
     @overload
     def read_file_stream(
         self,
-        file_id: Union[str, FileID],
+        file_id: str | FileID,
         encoding: Literal[None] = None,
-        errors: Optional[str] = None,
+        errors: str | None = None,
     ) -> Iterator[IO[bytes]]: ...
 
     @contextmanager
     @overload
     def read_file_stream(
-        self, file_id: Union[str, FileID], encoding: str, errors: Optional[str] = None
+        self, file_id: str | FileID, encoding: str, errors: str | None = None
     ) -> Iterator[IO[str]]: ...
 
     @contextmanager
     @overload
     def read_file_stream(
         self,
-        file_id: Union[str, FileID],
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-    ) -> Union[Iterator[IO[bytes]], Iterator[IO[str]]]: ...
+        file_id: str | FileID,
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> Iterator[IO[bytes]] | Iterator[IO[str]]: ...
 
     @contextmanager
     def read_file_stream(
         self,
-        file_id: Union[str, FileID],
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-    ) -> Union[Iterator[IO[bytes]], Iterator[IO[str]]]:
+        file_id: str | FileID,
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> Iterator[IO[bytes]] | Iterator[IO[str]]:
         self._check_job_store_file_id(file_id)
         if encoding is None:
             with open(
@@ -779,7 +779,7 @@ class FileJobStore(AbstractJobStore, URLAccess):
         self,
         shared_file_name: str,
         encoding: str,
-        errors: Optional[str] = None,
+        errors: str | None = None,
     ) -> Iterator[IO[str]]: ...
 
     @overload
@@ -788,16 +788,16 @@ class FileJobStore(AbstractJobStore, URLAccess):
         self,
         shared_file_name: str,
         encoding: Literal[None] = None,
-        errors: Optional[str] = None,
+        errors: str | None = None,
     ) -> Iterator[IO[bytes]]: ...
 
     @contextmanager
     def read_shared_file_stream(
         self,
         shared_file_name: str,
-        encoding: Optional[str] = None,
-        errors: Optional[str] = None,
-    ) -> Union[Iterator[IO[bytes]], Iterator[IO[str]]]:
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> Iterator[IO[bytes]] | Iterator[IO[str]]:
         self._requireValidSharedFileName(shared_file_name)
         try:
             with open(
@@ -814,7 +814,7 @@ class FileJobStore(AbstractJobStore, URLAccess):
             else:
                 raise
 
-    def list_all_file_names(self, for_job: Optional[str] = None) -> Iterable[str]:
+    def list_all_file_names(self, for_job: str | None = None) -> Iterable[str]:
         """
         Get all the file names (not file IDs) of files stored in the job store.
 
@@ -871,12 +871,18 @@ class FileJobStore(AbstractJobStore, URLAccess):
 
     def write_logs(self, msg):
         # Temporary files are placed in the stats directory tree
-        tempStatsFileName = self.LOG_PREFIX + str(uuid.uuid4().hex) + self.LOG_TEMP_SUFFIX
-        tempStatsFile = os.path.join(self._get_arbitrary_stats_inbox_dir(), tempStatsFileName)
+        tempStatsFileName = (
+            self.LOG_PREFIX + str(uuid.uuid4().hex) + self.LOG_TEMP_SUFFIX
+        )
+        tempStatsFile = os.path.join(
+            self._get_arbitrary_stats_inbox_dir(), tempStatsFileName
+        )
         writeFormat = "w" if isinstance(msg, str) else "wb"
         with open(tempStatsFile, writeFormat) as f:
             f.write(msg)
-        os.rename(tempStatsFile, tempStatsFile[:-len(self.LOG_TEMP_SUFFIX)])  # This operation is atomic
+        os.rename(
+            tempStatsFile, tempStatsFile[: -len(self.LOG_TEMP_SUFFIX)]
+        )  # This operation is atomic
 
     def read_logs(self, callback, read_all=False):
         files_processed = 0

toil/jobStores/googleJobStore.py

@@ -17,10 +17,11 @@ import pickle
 import stat
 import time
 import uuid
+from collections.abc import Iterator
 from contextlib import contextmanager
 from functools import wraps
 from io import BytesIO
-from typing import Any, IO, Iterator, Optional
+from typing import IO, Any
 from urllib.parse import ParseResult, urlunparse
 
 from google.api_core.exceptions import (
@@ -39,10 +40,10 @@ from toil.jobStores.abstractJobStore import (
     NoSuchJobException,
     NoSuchJobStoreException,
 )
-from toil.lib.pipes import ReadablePipe, WritablePipe
 from toil.lib.compatibility import compat_bytes
 from toil.lib.io import AtomicFileCreate
 from toil.lib.misc import truncExpBackoff
+from toil.lib.pipes import ReadablePipe, WritablePipe
 from toil.lib.retry import old_retry
 from toil.lib.url import URLAccess
 
@@ -92,6 +93,7 @@ def google_retry(f):
 
     return wrapper
 
+
 @contextmanager
 def permission_error_reporter(url: ParseResult, notes: str) -> Iterator[None]:
     """
@@ -102,7 +104,7 @@ def permission_error_reporter(url: ParseResult, notes: str) -> Iterator[None]:
     behind the scenes. Then it will complain::
 
         <class 'google.auth.exceptions.InvalidOperation'>: Anonymous credentials cannot be refreshed.
-    
+
     We need to detect this and report that the real problem is that the user
     has not set up any credentials. When you try to make the client
     non-anonymously and don't have credentials set up, you get a nice error
@@ -138,7 +140,6 @@ def permission_error_reporter(url: ParseResult, notes: str) -> Iterator[None]:
             raise
 
 
-
 class GoogleJobStore(AbstractJobStore, URLAccess):
 
     nodeServiceAccountJson = "/root/service_account.json"
@@ -182,6 +183,7 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
         """
 
         notes: list[str] = []
+
         def add_note(message: str, *args: Any, warn: bool = False) -> None:
             """
             Add and possibly warn with a note about the client permissions.
@@ -190,6 +192,7 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
             if warn:
                 log.warning(note)
             notes.append(note)
+
         def compile_notes() -> str:
             """
             Make one string explainign why we might not have expected permissions.
@@ -202,9 +205,7 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
         # Determine if we have an override environment variable for our credentials.
         # We get the path to check existence, but Google Storage works out what
         # to use later by looking at the environment again.
-        credentials_path: Optional[str] = os.getenv(
-            "GOOGLE_APPLICATION_CREDENTIALS", None
-        )
+        credentials_path: str | None = os.getenv("GOOGLE_APPLICATION_CREDENTIALS", None)
         if credentials_path is not None and not os.path.exists(credentials_path):
             # If the file is missing, complain.
             # This variable holds a file name and not any sensitive data itself.
@@ -212,22 +213,25 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
                 "File '%s' from GOOGLE_APPLICATION_CREDENTIALS is unavailable! "
                 "We may not be able to authenticate!",
                 credentials_path,
-                warn=True
+                warn=True,
             )
 
         if credentials_path is None and os.path.exists(cls.nodeServiceAccountJson):
             try:
                 # load credentials from a particular file on GCE nodes if an
                 # override path is not set
-                return storage.Client.from_service_account_json(
-                    cls.nodeServiceAccountJson
-                ), compile_notes()
+                return (
+                    storage.Client.from_service_account_json(
+                        cls.nodeServiceAccountJson
+                    ),
+                    compile_notes(),
+                )
             except OSError:
                 # Probably we don't have permission to use the file.
                 add_note(
                     "File '%s' exists but didn't work to authenticate!",
                     cls.nodeServiceAccountJson,
-                    warn=True
+                    warn=True,
                 )
 
         # Either a filename is specified, or our fallback file isn't there.
@@ -366,9 +370,7 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
 
         env = {}
 
-        credentials_path: Optional[str] = os.getenv(
-            "GOOGLE_APPLICATION_CREDENTIALS", None
-        )
+        credentials_path: str | None = os.getenv("GOOGLE_APPLICATION_CREDENTIALS", None)
         if credentials_path is not None:
             # Send along the environment variable that points to the credentials file.
             # It must be available in the same place on all nodes.
@@ -486,7 +488,9 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
 
     @classmethod
     @google_retry
-    def _get_blob_from_url(cls, client: storage.Client, url: ParseResult, exists: bool = False) -> storage.blob.Blob:
+    def _get_blob_from_url(
+        cls, client: storage.Client, url: ParseResult, exists: bool = False
+    ) -> storage.blob.Blob:
         """
         Gets the blob specified by the url.
 
@@ -521,7 +525,7 @@ class GoogleJobStore(AbstractJobStore, URLAccess):
     @classmethod
     def _url_exists(cls, url: ParseResult) -> bool:
         client, auth_notes = cls.create_client()
-        with permission_error_reporter(url, auth_notes): 
+        with permission_error_reporter(url, auth_notes):
             try:
                 cls._get_blob_from_url(client, url, exists=True)
                 return True

toil/jobStores/utils.py

@@ -1,15 +1,11 @@
-import errno
 import logging
 import os
 import tempfile
 import uuid
-from abc import ABC, abstractmethod
-from typing import Optional
-
-from toil.lib.threading import ExceptionalThread
 
 log = logging.getLogger(__name__)
 
+
 class JobStoreUnavailableException(RuntimeError):
     """
     Raised when a particular type of job store is requested but can't be used.
@@ -18,8 +14,8 @@ class JobStoreUnavailableException(RuntimeError):
 
 def generate_locator(
     job_store_type: str,
-    local_suggestion: Optional[str] = None,
-    decoration: Optional[str] = None,
+    local_suggestion: str | None = None,
+    decoration: str | None = None,
 ) -> str:
     """
     Generate a random locator for a job store of the given type. Raises an

toil/leader.py

@@ -21,7 +21,7 @@ import os
 import pickle
 import sys
 import time
-from typing import Any, Optional, Union
+from typing import Any
 
 import enlighten
 
@@ -89,10 +89,10 @@ class Leader:
         self,
         config: Config,
         batchSystem: AbstractBatchSystem,
-        provisioner: Optional[AbstractProvisioner],
+        provisioner: AbstractProvisioner | None,
         jobStore: AbstractJobStore,
         rootJob: JobDescription,
-        jobCache: Optional[dict[Union[str, TemporaryID], JobDescription]] = None,
+        jobCache: dict[str | TemporaryID, JobDescription] | None = None,
     ) -> None:
         """
         Create a Toil Leader object.
@@ -201,7 +201,7 @@ class Leader:
 
         # A dashboard that runs on the leader node in AWS clusters to track the state
         # of the cluster
-        self.toilMetrics: Optional[ToilMetrics] = None
+        self.toilMetrics: ToilMetrics | None = None
 
         # internal jobs we should not expose at top level debugging
         self.debugJobNames = (
@@ -349,14 +349,9 @@ class Leader:
     def create_status_sentinel_file(self, fail: bool) -> None:
         """Create a file in the jobstore indicating failure or success."""
         logName = "failed.log" if fail else "succeeded.log"
-        localLog = os.path.join(os.getcwd(), logName)
-        open(localLog, "w").close()
-        self.jobStore.import_file("file://" + localLog, logName, hardlink=True)
-
-        if os.path.exists(
-            localLog
-        ):  # Bandaid for Jenkins tests failing stochastically and unexplainably.
-            os.remove(localLog)
+        with self.jobStore.write_shared_file_stream(logName) as file_handle:
+            # We just need an empty file, so don't write any content
+            pass
 
     def _handledFailedSuccessor(self, successor_id: str, predecessor_id: str) -> bool:
         """
@@ -852,15 +847,15 @@ class Leader:
                 )
                 message = [
                     f"Job failed with exit value {status_string}: {updatedJob}",
-                    f"Exit reason: {BatchJobExitReason.to_string(update.exitReason)}"
+                    f"Exit reason: {BatchJobExitReason.to_string(update.exitReason)}",
                 ]
                 if update.backing_id is not None:
                     # Report the job in the backing scheduler in case the user
                     # needs to follow it down a level.
-                    message.append(f"Failed job in backing scheduler: {update.backing_id}")
-                logger.warning(
-                    "\n".join(message)
-                )
+                    message.append(
+                        f"Failed job in backing scheduler: {update.backing_id}"
+                    )
+                logger.warning("\n".join(message))
                 # This logic is undefined for which of the failing jobs will send its exit code
                 # when there are multiple failing jobs with different exit statuses
                 self.recommended_fail_exit_code = update.exitStatus
@@ -1188,7 +1183,7 @@ class Leader:
             )
             self.preemptibleServiceJobsIssued += 1
 
-    def getNumberOfJobsIssued(self, preemptible: Optional[bool] = None) -> int:
+    def getNumberOfJobsIssued(self, preemptible: bool | None = None) -> int:
         """
         Get number of jobs that have been added by issueJob(s) and not removed by removeJob.
 
@@ -1267,7 +1262,7 @@ class Leader:
 
         return issuedDesc
 
-    def getJobs(self, preemptible: Optional[bool] = None) -> list[JobDescription]:
+    def getJobs(self, preemptible: bool | None = None) -> list[JobDescription]:
         """
         Get all issued jobs.
 
@@ -1427,9 +1422,9 @@ class Leader:
         self,
         finished_job: JobDescription,
         result_status: int,
-        wall_time: Optional[float] = None,
-        exit_reason: Optional[BatchJobExitReason] = None,
-        batch_system_id: Optional[int] = None,
+        wall_time: float | None = None,
+        exit_reason: BatchJobExitReason | None = None,
+        batch_system_id: int | None = None,
     ) -> bool:
         """
         Process a finished JobDescription based upon its success or failure.

toil/lib/accelerators.py

@@ -17,7 +17,7 @@
 import os
 import string
 import subprocess
-from typing import Union, cast
+from typing import cast
 from xml.dom import minidom
 
 from toil.job import AcceleratorRequirement
@@ -34,7 +34,9 @@ def have_working_nvidia_smi() -> bool:
     it can fulfill a CUDARequirement.
     """
     try:
-        subprocess.check_call(["nvidia-smi"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        subprocess.check_call(
+            ["nvidia-smi"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
     except (
         FileNotFoundError,
         PermissionError,
@@ -105,7 +107,7 @@ def have_working_nvidia_docker_runtime() -> bool:
                 "nvidia-smi",
             ],
             stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL
+            stderr=subprocess.DEVNULL,
         )
     except (
         FileNotFoundError,
@@ -217,7 +219,7 @@ def get_individual_local_accelerators() -> list[AcceleratorRequirement]:
 
 
 def get_restrictive_environment_for_local_accelerators(
-    accelerator_numbers: Union[set[int], list[int]]
+    accelerator_numbers: set[int] | list[int],
 ) -> dict[str, str]:
     """
     Get environment variables which can be applied to a process to restrict it

toil/lib/aws/__init__.py

@@ -15,10 +15,9 @@ import json
 import logging
 import os
 import re
-import socket
 from collections.abc import MutableMapping
 from http.client import HTTPException
-from typing import TYPE_CHECKING, Literal, Optional, Union
+from typing import TYPE_CHECKING, Literal, Union
 from urllib.error import URLError
 from urllib.request import urlopen
 
@@ -43,7 +42,7 @@ logger = logging.getLogger(__name__)
 # which may not be installed, because it has to be importable everywhere.
 
 
-def get_current_aws_region() -> Optional[str]:
+def get_current_aws_region() -> str | None:
     """
     Return the AWS region that the currently configured AWS zone (see
     get_current_aws_zone()) is in.
@@ -53,14 +52,14 @@ def get_current_aws_region() -> Optional[str]:
     return zone_to_region(aws_zone) if aws_zone else None
 
 
-def get_aws_zone_from_environment() -> Optional[str]:
+def get_aws_zone_from_environment() -> str | None:
     """
     Get the AWS zone from TOIL_AWS_ZONE if set.
     """
     return os.environ.get("TOIL_AWS_ZONE", None)
 
 
-def get_aws_zone_from_metadata() -> Optional[str]:
+def get_aws_zone_from_metadata() -> str | None:
     """
     Get the AWS zone from instance metadata, if on EC2 and the boto module is
     available. Otherwise, gets the AWS zone from ECS task metadata, if on ECS.
@@ -103,7 +102,7 @@ def get_aws_zone_from_metadata() -> Optional[str]:
     return None
 
 
-def get_aws_zone_from_boto() -> Optional[str]:
+def get_aws_zone_from_boto() -> str | None:
     """
     Get the AWS zone from the Boto3 config file or from AWS_DEFAULT_REGION, if it is configured and the
     boto3 module is available.
@@ -122,7 +121,7 @@ def get_aws_zone_from_boto() -> Optional[str]:
     return None
 
 
-def get_aws_zone_from_environment_region() -> Optional[str]:
+def get_aws_zone_from_environment_region() -> str | None:
     """
     Pick an AWS zone in the region defined by TOIL_AWS_REGION, if it is set.
     """
@@ -134,7 +133,7 @@ def get_aws_zone_from_environment_region() -> Optional[str]:
     return None
 
 
-def get_current_aws_zone() -> Optional[str]:
+def get_current_aws_zone() -> str | None:
     """
     Get the currently configured or occupied AWS zone to use.
 
@@ -191,7 +190,7 @@ def running_on_ec2() -> bool:
             timeout=1,
         )
         return True
-    except (URLError, socket.timeout, HTTPException):
+    except (URLError, TimeoutError, HTTPException):
         return False
 
 
@@ -204,7 +203,7 @@ def running_on_ecs() -> bool:
 
 
 def build_tag_dict_from_env(
-    environment: MutableMapping[str, str] = os.environ
+    environment: MutableMapping[str, str] = os.environ,
 ) -> dict[str, str]:
     tags = dict()
     owner_tag = environment.get("TOIL_OWNER_TAG")