toil 9.0.0__py3-none-any.whl → 9.1.1__py3-none-any.whl

toil/jobStores/utils.py

@@ -10,333 +10,6 @@ from toil.lib.threading import ExceptionalThread
 
 log = logging.getLogger(__name__)
 
-
-class WritablePipe(ABC):
-    """
-    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
-    :meth:`.readFrom` to consume the readable end of the pipe, then instantiate the class as a
-    context manager to get the writable end. See the example below.
-
-    >>> import sys, shutil
-    >>> class MyPipe(WritablePipe):
-    ...     def readFrom(self, readable):
-    ...         shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
-    >>> with MyPipe() as writable:
-    ...     _ = writable.write('Hello, world!\\n'.encode('utf-8'))
-    Hello, world!
-
-    Each instance of this class creates a thread and invokes the readFrom method in that thread.
-    The thread will be join()ed upon normal exit from the context manager, i.e. the body of the
-    `with` statement. If an exception occurs, the thread will not be joined but a well-behaved
-    :meth:`.readFrom` implementation will terminate shortly thereafter due to the pipe having
-    been closed.
-
-    Now, exceptions in the reader thread will be reraised in the main thread:
-
-    >>> class MyPipe(WritablePipe):
-    ...     def readFrom(self, readable):
-    ...         raise RuntimeError('Hello, world!')
-    >>> with MyPipe() as writable:
-    ...     pass
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-
-    More complicated, less illustrative tests:
-
-    Same as above, but proving that handles are closed:
-
-    >>> x = os.dup(0); os.close(x)
-    >>> class MyPipe(WritablePipe):
-    ...     def readFrom(self, readable):
-    ...         raise RuntimeError('Hello, world!')
-    >>> with MyPipe() as writable:
-    ...     pass
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-    >>> y = os.dup(0); os.close(y); x == y
-    True
-
-    Exceptions in the body of the with statement aren't masked, and handles are closed:
-
-    >>> x = os.dup(0); os.close(x)
-    >>> class MyPipe(WritablePipe):
-    ...     def readFrom(self, readable):
-    ...         pass
-    >>> with MyPipe() as writable:
-    ...     raise RuntimeError('Hello, world!')
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-    >>> y = os.dup(0); os.close(y); x == y
-    True
-    """
-
-    @abstractmethod
-    def readFrom(self, readable):
-        """
-        Implement this method to read data from the pipe. This method should support both
-        binary and text mode output.
-
-        :param file readable: the file object representing the readable end of the pipe. Do not
-            explicitly invoke the close() method of the object, that will be done automatically.
-        """
-        raise NotImplementedError()
-
-    def _reader(self):
-        with os.fdopen(self.readable_fh, "rb") as readable:
-            # TODO: If the reader somehow crashes here, both threads might try
-            # to close readable_fh.  Fortunately we don't do anything that
-            # should be able to fail here.
-            self.readable_fh = None  # signal to parent thread that we've taken over
-            self.readFrom(readable)
-            self.reader_done = True
-
-    def __init__(self, encoding=None, errors=None):
-        """
-        The specified encoding and errors apply to the writable end of the pipe.
-
-        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
-                as for encode(). Defaults to None which represents binary mode.
-
-        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
-                are the same as for open(). Defaults to 'strict' when an encoding is specified.
-        """
-        super().__init__()
-        self.encoding = encoding
-        self.errors = errors
-        self.readable_fh = None
-        self.writable = None
-        self.thread = None
-        self.reader_done = False
-
-    def __enter__(self):
-        self.readable_fh, writable_fh = os.pipe()
-        self.writable = os.fdopen(
-            writable_fh,
-            "wb" if self.encoding == None else "wt",
-            encoding=self.encoding,
-            errors=self.errors,
-        )
-        self.thread = ExceptionalThread(target=self._reader)
-        self.thread.start()
-        return self.writable
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        # Closing the writable end will send EOF to the readable and cause the reader thread
-        # to finish.
-        # TODO: Can close() fail? If so, would we try and clean up after the reader?
-        self.writable.close()
-        try:
-            if self.thread is not None:
-                # reraises any exception that was raised in the thread
-                self.thread.join()
-        except Exception as e:
-            if exc_type is None:
-                # Only raise the child exception if there wasn't
-                # already an exception in the main thread
-                raise
-            else:
-                log.error(
-                    "Swallowing additional exception in reader thread: %s", str(e)
-                )
-        finally:
-            # The responsibility for closing the readable end is generally that of the reader
-            # thread. To cover the small window before the reader takes over we also close it here.
-            readable_fh = self.readable_fh
-            if readable_fh is not None:
-                # Close the file handle. The reader thread must be dead now.
-                os.close(readable_fh)
-
-
-class ReadablePipe(ABC):
-    """
-    An object-oriented wrapper for os.pipe. Clients should subclass it, implement
-    :meth:`.writeTo` to place data into the writable end of the pipe, then instantiate the class
-    as a context manager to get the writable end. See the example below.
-
-    >>> import sys, shutil
-    >>> class MyPipe(ReadablePipe):
-    ...     def writeTo(self, writable):
-    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
-    >>> with MyPipe() as readable:
-    ...     shutil.copyfileobj(codecs.getreader('utf-8')(readable), sys.stdout)
-    Hello, world!
-
-    Each instance of this class creates a thread and invokes the :meth:`.writeTo` method in that
-    thread. The thread will be join()ed upon normal exit from the context manager, i.e. the body
-    of the `with` statement. If an exception occurs, the thread will not be joined but a
-    well-behaved :meth:`.writeTo` implementation will terminate shortly thereafter due to the
-    pipe having been closed.
-
-    Now, exceptions in the reader thread will be reraised in the main thread:
-
-    >>> class MyPipe(ReadablePipe):
-    ...     def writeTo(self, writable):
-    ...         raise RuntimeError('Hello, world!')
-    >>> with MyPipe() as readable:
-    ...     pass
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-
-    More complicated, less illustrative tests:
-
-    Same as above, but proving that handles are closed:
-
-    >>> x = os.dup(0); os.close(x)
-    >>> class MyPipe(ReadablePipe):
-    ...     def writeTo(self, writable):
-    ...         raise RuntimeError('Hello, world!')
-    >>> with MyPipe() as readable:
-    ...     pass
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-    >>> y = os.dup(0); os.close(y); x == y
-    True
-
-    Exceptions in the body of the with statement aren't masked, and handles are closed:
-
-    >>> x = os.dup(0); os.close(x)
-    >>> class MyPipe(ReadablePipe):
-    ...     def writeTo(self, writable):
-    ...         pass
-    >>> with MyPipe() as readable:
-    ...     raise RuntimeError('Hello, world!')
-    Traceback (most recent call last):
-    ...
-    RuntimeError: Hello, world!
-    >>> y = os.dup(0); os.close(y); x == y
-    True
-    """
-
-    @abstractmethod
-    def writeTo(self, writable):
-        """
-        Implement this method to write data from the pipe. This method should support both
-        binary and text mode input.
-
-        :param file writable: the file object representing the writable end of the pipe. Do not
-            explicitly invoke the close() method of the object, that will be done automatically.
-        """
-        raise NotImplementedError()
-
-    def _writer(self):
-        try:
-            with os.fdopen(self.writable_fh, "wb") as writable:
-                self.writeTo(writable)
-        except OSError as e:
-            # The other side of the pipe may have been closed by the
-            # reading thread, which is OK.
-            if e.errno != errno.EPIPE:
-                raise
-
-    def __init__(self, encoding=None, errors=None):
-        """
-        The specified encoding and errors apply to the readable end of the pipe.
-
-        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
-                as for encode(). Defaults to None which represents binary mode.
-
-        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
-                are the same as for open(). Defaults to 'strict' when an encoding is specified.
-        """
-        super().__init__()
-        self.encoding = encoding
-        self.errors = errors
-        self.writable_fh = None
-        self.readable = None
-        self.thread = None
-
-    def __enter__(self):
-        readable_fh, self.writable_fh = os.pipe()
-        self.readable = os.fdopen(
-            readable_fh,
-            "rb" if self.encoding == None else "rt",
-            encoding=self.encoding,
-            errors=self.errors,
-        )
-        self.thread = ExceptionalThread(target=self._writer)
-        self.thread.start()
-        return self.readable
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        # Close the read end of the pipe. The writing thread may
-        # still be writing to the other end, but this will wake it up
-        # if that's the case.
-        self.readable.close()
-        try:
-            if self.thread is not None:
-                # reraises any exception that was raised in the thread
-                self.thread.join()
-        except:
-            if exc_type is None:
-                # Only raise the child exception if there wasn't
-                # already an exception in the main thread
-                raise
-
-
-class ReadableTransformingPipe(ReadablePipe):
-    """
-    A pipe which is constructed around a readable stream, and which provides a
-    context manager that gives a readable stream.
-
-    Useful as a base class for pipes which have to transform or otherwise visit
-    bytes that flow through them, instead of just consuming or producing data.
-
-    Clients should subclass it and implement :meth:`.transform`, like so:
-
-    >>> import sys, shutil
-    >>> class MyPipe(ReadableTransformingPipe):
-    ...     def transform(self, readable, writable):
-    ...         writable.write(readable.read().decode('utf-8').upper().encode('utf-8'))
-    >>> class SourcePipe(ReadablePipe):
-    ...     def writeTo(self, writable):
-    ...         writable.write('Hello, world!\\n'.encode('utf-8'))
-    >>> with SourcePipe() as source:
-    ...     with MyPipe(source) as transformed:
-    ...         shutil.copyfileobj(codecs.getreader('utf-8')(transformed), sys.stdout)
-    HELLO, WORLD!
-
-    The :meth:`.transform` method runs in its own thread, and should move data
-    chunk by chunk instead of all at once. It should finish normally if it
-    encounters either an EOF on the readable, or a :class:`BrokenPipeError` on
-    the writable. This means that it should make sure to actually catch a
-    :class:`BrokenPipeError` when writing.
-
-    See also: :class:`toil.lib.misc.WriteWatchingStream`.
-
-    """
-
-    def __init__(self, source, encoding=None, errors=None):
-        """
-        :param str encoding: the name of the encoding used to encode the file. Encodings are the same
-                as for encode(). Defaults to None which represents binary mode.
-
-        :param str errors: an optional string that specifies how encoding errors are to be handled. Errors
-                are the same as for open(). Defaults to 'strict' when an encoding is specified.
-        """
-        super().__init__(encoding=encoding, errors=errors)
-        self.source = source
-
-    @abstractmethod
-    def transform(self, readable, writable):
-        """
-        Implement this method to ship data through the pipe.
-
-        :param file readable: the input stream file object to transform.
-
-        :param file writable: the file object representing the writable end of the pipe. Do not
-            explicitly invoke the close() method of the object, that will be done automatically.
-        """
-        raise NotImplementedError()
-
-    def writeTo(self, writable):
-        self.transform(self.source, writable)
-
-
 class JobStoreUnavailableException(RuntimeError):
     """
     Raised when a particular type of job store is requested but can't be used.

toil/leader.py

@@ -31,6 +31,7 @@ from toil.batchSystems.abstractBatchSystem import (
     EXIT_STATUS_UNAVAILABLE_VALUE,
     AbstractBatchSystem,
     BatchJobExitReason,
+    UpdatedBatchJobInfo,
 )
 from toil.bus import (
     JobCompletedMessage,
@@ -829,40 +830,41 @@ class Leader:
             # Mark the service job updated so we don't stop here.
             self._messages.publish(JobUpdatedMessage(client_id, 1))
 
-    def _gatherUpdatedJobs(self, updatedJobTuple):
+    def _gatherUpdatedJobs(self, update: UpdatedBatchJobInfo):
         """Gather any new, updated JobDescriptions from the batch system."""
-        bsID, exitStatus, exitReason, wallTime = (
-            updatedJobTuple.jobID,
-            updatedJobTuple.exitStatus,
-            updatedJobTuple.exitReason,
-            updatedJobTuple.wallTime,
-        )
         # easy, track different state
         try:
             updatedJob = self.toilState.get_job(
-                self.issued_jobs_by_batch_system_id[bsID]
+                self.issued_jobs_by_batch_system_id[update.jobID]
             )
         except KeyError:
             logger.warning(
-                "A result seems to already have been processed for job %s", bsID
+                "A result seems to already have been processed for job %s", update.jobID
             )
         else:
-            if exitStatus == 0:
+            if update.exitStatus == 0:
                 logger.debug("Job ended: %s", updatedJob)
             else:
                 status_string = (
-                    str(exitStatus)
-                    if exitStatus != EXIT_STATUS_UNAVAILABLE_VALUE
+                    str(update.exitStatus)
+                    if update.exitStatus != EXIT_STATUS_UNAVAILABLE_VALUE
                     else "<UNAVAILABLE>"
                 )
+                message = [
+                    f"Job failed with exit value {status_string}: {updatedJob}",
+                    f"Exit reason: {BatchJobExitReason.to_string(update.exitReason)}"
+                ]
+                if update.backing_id is not None:
+                    # Report the job in the backing scheduler in case the user
+                    # needs to follow it down a level.
+                    message.append(f"Failed job in backing scheduler: {update.backing_id}")
                 logger.warning(
-                    f"Job failed with exit value {status_string}: {updatedJob}\n"
-                    f"Exit reason: {BatchJobExitReason.to_string(exitReason)}"
+                    "\n".join(message)
                 )
                 # This logic is undefined for which of the failing jobs will send its exit code
                 # when there are multiple failing jobs with different exit statuses
-                self.recommended_fail_exit_code = exitStatus
-                if exitStatus == CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE:
+                self.recommended_fail_exit_code = update.exitStatus
+                if update.exitStatus == CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE:
                     # This is a CWL job informing us that the workflow is
                     # asking things of us that Toil can't do. When we raise an
                     # exception because of this, make sure to forward along
@@ -876,11 +878,14 @@ class Leader:
                 JobCompletedMessage(
                     get_job_kind(updatedJob.get_names()),
                     updatedJob.jobStoreID,
-                    exitStatus,
+                    update.exitStatus,
                 )
             )
             self.process_finished_job(
-                bsID, exitStatus, wall_time=wallTime, exit_reason=exitReason
+                update.jobID,
+                update.exitStatus,
+                wall_time=update.wallTime,
+                exit_reason=update.exitReason,
             )
 
     def _processLostJobs(self):
@@ -926,10 +931,10 @@ class Leader:
             self._processJobsWithFailedServices()
 
             # check in with the batch system
-            updatedJobTuple = self.batchSystem.getUpdatedBatchJob(maxWait=2)
-            if updatedJobTuple is not None:
+            update: UpdatedBatchJobInfo = self.batchSystem.getUpdatedBatchJob(maxWait=2)
+            if update is not None:
                 # Collect and process all the updates
-                self._gatherUpdatedJobs(updatedJobTuple)
+                self._gatherUpdatedJobs(update)
                 # As long as we are getting updates we definitely can't be
                 # deadlocked.
                 self.feed_deadlock_watchdog()
@@ -1125,7 +1130,7 @@ class Leader:
             self.preemptibleJobsIssued += 1
         cur_logger = logger.debug if jobNode.local else logger.info
         cur_logger(
-            "Issued job %s with job batch system ID: " "%s and %s",
+            "Issued job %s with job batch system ID: %s and %s",
             jobNode,
             str(jobBatchSystemID),
             jobNode.requirements_string(),

toil/lib/aws/config.py

@@ -0,0 +1,22 @@
+S3_PARALLELIZATION_FACTOR = 8
+S3_PART_SIZE = 16 * 1024 * 1024
+KiB = 1024
+MiB = KiB * KiB
+
+# Files must be larger than this before we consider multipart uploads.
+AWS_MIN_CHUNK_SIZE = 64 * MiB
+# Convenience variable for Boto3 TransferConfig(multipart_threhold=).
+MULTIPART_THRESHOLD = AWS_MIN_CHUNK_SIZE + 1
+# Maximum number of parts allowed in a multipart upload.  This is a limitation imposed by S3.
+AWS_MAX_MULTIPART_COUNT = 10000
+
+
+def get_s3_multipart_chunk_size(filesize: int) -> int:
+    """Returns the chunk size of the S3 multipart object, given a file's size in bytes."""
+    if filesize <= AWS_MAX_MULTIPART_COUNT * AWS_MIN_CHUNK_SIZE:
+        return AWS_MIN_CHUNK_SIZE
+    else:
+        div = filesize // AWS_MAX_MULTIPART_COUNT
+        if div * AWS_MAX_MULTIPART_COUNT < filesize:
+            div += 1
+        return ((div + MiB - 1) // MiB) * MiB