toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/jobStores/abstractJobStore.py

@@ -27,6 +27,7 @@ from typing import (IO,
                     Callable,
                     ContextManager,
                     Dict,
+                    Iterable,
                     Iterator,
                     List,
                     Optional,
@@ -42,8 +43,8 @@ if sys.version_info >= (3, 8):
 else:
     from typing_extensions import Literal
 
-from urllib.parse import ParseResult, urlparse
 from urllib.error import HTTPError
+from urllib.parse import ParseResult, urlparse
 from urllib.request import urlopen
 from uuid import uuid4
 
@@ -536,6 +537,40 @@ class AbstractJobStore(ABC):
                 executable = jobStoreFileID.executable
             otherCls._write_to_url(readable, url, executable)
 
+    @classmethod
+    def url_exists(cls, src_uri: str) -> bool:
+        """
+        Return True if the file at the given URI exists, and False otherwise.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._findJobStoreForUrl(parseResult)
+        return otherCls._url_exists(parseResult)
+
+    @classmethod
+    def get_size(cls, src_uri: str) -> Optional[int]:
+        """
+        Get the size in bytes of the file at the given URL, or None if it cannot be obtained.
+
+        :param src_uri: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._findJobStoreForUrl(parseResult)
+        return otherCls._get_size(parseResult)
+
+    @classmethod
+    def get_is_directory(cls, src_uri: str) -> bool:
+        """
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file. The URL may or may not end in '/'.
+        """
+        parseResult = urlparse(src_uri)
+        otherCls = cls._findJobStoreForUrl(parseResult)
+        return otherCls._get_is_directory(parseResult)
+
     @classmethod
     def list_url(cls, src_uri: str) -> List[str]:
         """
@@ -562,59 +597,61 @@ class AbstractJobStore(ABC):
         return otherCls._list_url(parseResult)
 
     @classmethod
-    def get_is_directory(cls, src_uri: str) -> bool:
-        """
-        Return True if the thing at the given URL is a directory, and False if
-        it is a file. The URL may or may not end in '/'.
+    def read_from_url(cls, src_uri: str, writable: IO[bytes]) -> Tuple[int, bool]:
         """
-        parseResult = urlparse(src_uri)
-        otherCls = cls._findJobStoreForUrl(parseResult)
-        return otherCls._get_is_directory(parseResult)
+        Read the given URL and write its content into the given writable stream.
 
-    @classmethod
-    @abstractmethod
-    def _get_is_directory(cls, url: ParseResult) -> bool:
-        """
-        Return True if the thing at the given URL is a directory, and False if
-        it is a file or it is known not to exist. The URL may or may not end in
-        '/'.
+        Raises FileNotFoundError if the URL doesn't exist.
 
-        :param url: URL that points to a file or object, or directory or prefix,
-               in the storage mechanism of a supported URL scheme e.g. a blob
-               in an AWS s3 bucket.
+        :return: The size of the file in bytes and whether the executable permission bit is set
         """
-        raise NotImplementedError
+        parseResult = urlparse(src_uri)
+        otherCls = cls._findJobStoreForUrl(parseResult)
+        return otherCls._read_from_url(parseResult, writable)
 
     @classmethod
-    def read_from_url(cls, src_uri: str, writable: IO[bytes]) -> Tuple[int, bool]:
+    def open_url(cls, src_uri: str) -> IO[bytes]:
         """
-        Read the given URL and write its content into the given writable stream.
+        Read from the given URI.
 
         Raises FileNotFoundError if the URL doesn't exist.
 
-        :return: The size of the file in bytes and whether the executable permission bit is set
-        :rtype: Tuple[int, bool]
+        Has a readable stream interface, unlike :meth:`read_from_url` which
+        takes a writable stream.
         """
         parseResult = urlparse(src_uri)
         otherCls = cls._findJobStoreForUrl(parseResult)
-        return otherCls._read_from_url(parseResult, writable)
+        return otherCls._open_url(parseResult)
 
     @classmethod
-    @deprecated(new_function_name='get_size')
-    def getSize(cls, url: ParseResult) -> None:
-        return cls.get_size(url)
+    @abstractmethod
+    def _url_exists(cls, url: ParseResult) -> bool:
+        """
+        Return True if the item at the given URL exists, and Flase otherwise.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
-    def get_size(cls, src_uri: ParseResult) -> None:
+    def _get_size(cls, url: ParseResult) -> Optional[int]:
         """
-        Get the size in bytes of the file at the given URL, or None if it cannot be obtained.
+        Get the size of the object at the given URL, or None if it cannot be obtained.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
 
-        :param src_uri: URL that points to a file or object in the storage
-               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+    @classmethod
+    @abstractmethod
+    def _get_is_directory(cls, url: ParseResult) -> bool:
         """
-        raise NotImplementedError
+        Return True if the thing at the given URL is a directory, and False if
+        it is a file or it is known not to exist. The URL may or may not end in
+        '/'.
 
+        :param url: URL that points to a file or object, or directory or prefix,
+               in the storage mechanism of a supported URL scheme e.g. a blob
+               in an AWS s3 bucket.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -623,8 +660,6 @@ class AbstractJobStore(ABC):
         Reads the contents of the object at the specified location and writes it to the given
         writable stream.
 
-        Raises FileNotFoundError if the URL doesn't exist.
-
         Refer to :func:`~AbstractJobStore.importFile` documentation for currently supported URL schemes.
 
         Raises FileNotFoundError if the thing at the URL is not found.
@@ -635,46 +670,58 @@ class AbstractJobStore(ABC):
         :param IO[bytes] writable: a writable stream
 
         :return: The size of the file in bytes and whether the executable permission bit is set
-        :rtype: Tuple[int, bool]
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
-    def _write_to_url(cls, readable: Union[IO[bytes], IO[str]], url: ParseResult, executable: bool = False) -> None:
+    def _list_url(cls, url: ParseResult) -> List[str]:
         """
-        Reads the contents of the given readable stream and writes it to the object at the
-        specified location. Raises FileNotFoundError if the URL doesn't exist..
+        List the contents of the given URL, which may or may not end in '/'
 
-        Refer to AbstractJobStore.importFile documentation for currently supported URL schemes.
+        Returns a list of URL components. Those that end in '/' are meant to be
+        directories, while those that do not are meant to be files.
 
-        :param Union[IO[bytes], IO[str]] readable: a readable stream
+        Refer to :func:`~AbstractJobStore.importFile` documentation for currently supported URL schemes.
 
-        :param ParseResult url: URL that points to a file or object in the storage
-               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+        :param ParseResult url: URL that points to a directory or prefix in the
+        storage mechanism of a supported URL scheme e.g. a prefix in an AWS s3
+        bucket.
 
-        :param bool executable: determines if the file has executable permissions
+        :return: The children of the given URL, already URL-encoded if
+        appropriate. (If the URL is a bare path, no encoding is done.)
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
-    def _list_url(cls, url: ParseResult) -> List[str]:
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
         """
-        List the contents of the given URL, which may or may not end in '/'
-
-        Returns a list of URL components. Those that end in '/' are meant to be
-        directories, while those that do not are meant to be files.
+        Get a stream of the object at the specified location.
 
         Refer to :func:`~AbstractJobStore.importFile` documentation for currently supported URL schemes.
 
-        :param ParseResult url: URL that points to a directory or prefix in the
-        storage mechanism of a supported URL scheme e.g. a prefix in an AWS s3
-        bucket.
+        Raises FileNotFoundError if the thing at the URL is not found.
+        """
+        raise NotImplementedError(f"No implementation for {url}")
 
-        :return: The children of the given URL, already URL-encoded.
+    @classmethod
+    @abstractmethod
+    def _write_to_url(cls, readable: Union[IO[bytes], IO[str]], url: ParseResult, executable: bool = False) -> None:
         """
-        raise NotImplementedError()
+        Reads the contents of the given readable stream and writes it to the object at the
+        specified location. Raises FileNotFoundError if the URL doesn't exist..
+
+        Refer to AbstractJobStore.importFile documentation for currently supported URL schemes.
+
+        :param Union[IO[bytes], IO[str]] readable: a readable stream
+
+        :param ParseResult url: URL that points to a file or object in the storage
+               mechanism of a supported URL scheme e.g. a blob in an AWS s3 bucket.
+
+        :param bool executable: determines if the file has executable permissions
+        """
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -690,7 +737,7 @@ class AbstractJobStore(ABC):
 
         :return bool: returns true if the cls supports the URL
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @abstractmethod
     def destroy(self) -> None:
@@ -788,12 +835,17 @@ class AbstractJobStore(ABC):
             root_job_description = self.load_root_job()
             reachable_from_root: Set[str] = set()
 
-            # Add first root job outside of the loop below.
-            reachable_from_root.add(str(root_job_description.jobStoreID))
+            
+            for merged_in in root_job_description.get_chain():
+                # Add the job itself and any other jobs that chained with it.
+                # Keep merged-in jobs around themselves, but don't bother
+                # exploring them, since we took their successors.
+                reachable_from_root.add(merged_in.job_store_id)
             # add all of root's linked service jobs as well
-            for service_jobstore_id in root_job_description.services:
-                if haveJob(service_jobstore_id):
-                    reachable_from_root.add(service_jobstore_id)
+            for service_job_store_id in root_job_description.services:
+                if haveJob(service_job_store_id):
+                    reachable_from_root.add(service_job_store_id)
+            
 
             # Unprocessed means it might have successor jobs we need to add.
             unprocessed_job_descriptions = [root_job_description]
@@ -801,18 +853,19 @@ class AbstractJobStore(ABC):
             while unprocessed_job_descriptions:
                 new_job_descriptions_to_process = []  # Reset.
                 for job_description in unprocessed_job_descriptions:
-                    for successor_jobstore_id in job_description.allSuccessors():
-                        if successor_jobstore_id not in reachable_from_root and haveJob(successor_jobstore_id):
-                            successor_job_description = getJobDescription(successor_jobstore_id)
-
-                            # Add each successor job.
-                            reachable_from_root.add(
-                                str(successor_job_description.jobStoreID)
-                            )
+                    for merged_in in job_description.get_chain():
+                        # Add the job and anything chained with it.
+                        # Keep merged-in jobs around themselves, but don't bother
+                        # exploring them, since we took their successors.
+                        reachable_from_root.add(merged_in.job_store_id)
+                    for successor_job_store_id in job_description.allSuccessors():
+                        if successor_job_store_id not in reachable_from_root and haveJob(successor_job_store_id):
+                            successor_job_description = getJobDescription(successor_job_store_id)
+
                             # Add all of the successor's linked service jobs as well.
-                            for service_jobstore_id in successor_job_description.services:
-                                if haveJob(service_jobstore_id):
-                                    reachable_from_root.add(service_jobstore_id)
+                            for service_job_store_id in successor_job_description.services:
+                                if haveJob(service_job_store_id):
+                                    reachable_from_root.add(service_job_store_id)
 
                             new_job_descriptions_to_process.append(successor_job_description)
                 unprocessed_job_descriptions = new_job_descriptions_to_process
@@ -824,8 +877,8 @@ class AbstractJobStore(ABC):
 
         # Cleanup jobs that are not reachable from the root, and therefore orphaned
         # TODO: Avoid reiterating reachable_from_root (which may be very large)
-        jobsToDelete = [x for x in getJobDescriptions() if x.jobStoreID not in reachable_from_root]
-        for jobDescription in jobsToDelete:
+        unreachable = [x for x in getJobDescriptions() if x.jobStoreID not in reachable_from_root]
+        for jobDescription in unreachable:
             # clean up any associated files before deletion
             for fileID in jobDescription.filesToDelete:
                 # Delete any files that should already be deleted
@@ -1688,6 +1741,16 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
     def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
         return url.scheme.lower() in ('http', 'https', 'ftp') and not export
 
+    @classmethod
+    def _url_exists(cls, url: ParseResult) -> bool:
+        try:
+            # TODO: Figure out how to HEAD instead of this.
+            with cls._open_url(url):
+                return True
+        except:
+            pass
+        return False
+
     @classmethod
     @retry(
         errors=[
@@ -1695,7 +1758,7 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
             ErrorCondition(error=HTTPError, error_codes=[408, 500, 503]),
         ]
     )
-    def get_size(cls, url: ParseResult) -> Optional[int]:
+    def _get_size(cls, url: ParseResult) -> Optional[int]:
         if url.scheme.lower() == 'ftp':
             return None
         with closing(urlopen(url.geturl())) as readable:
@@ -1703,6 +1766,27 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
             size = readable.info().get('content-length')
             return int(size) if size is not None else None
 
+    @classmethod
+    def _read_from_url(
+        cls, url: ParseResult, writable: Union[IO[bytes], IO[str]]
+    ) -> Tuple[int, bool]:
+        # We can't actually retry after we start writing.
+        # TODO: Implement retry with byte range requests
+        with cls._open_url(url) as readable:
+            # Make something to count the bytes we get
+            # We need to put the actual count in a container so our
+            # nested function can modify it without creating its own
+            # local with the same name.
+            size = [0]
+            def count(l: int) -> None:
+                size[0] += l
+            counter = WriteWatchingStream(writable)
+            counter.onWrite(count)
+
+            # Do the download
+            shutil.copyfileobj(readable, counter)
+            return size[0], False
+
     @classmethod
     @retry(
         errors=[
@@ -1710,27 +1794,9 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
             ErrorCondition(error=HTTPError, error_codes=[408, 500, 503]),
         ]
     )
-    def _read_from_url(
-        cls, url: ParseResult, writable: Union[IO[bytes], IO[str]]
-    ) -> Tuple[int, bool]:
-        # We can only retry on errors that happen as responses to the request.
-        # If we start getting file data, and the connection drops, we fail.
-        # So we don't have to worry about writing the start of the file twice.
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
         try:
-            with closing(urlopen(url.geturl())) as readable:
-                # Make something to count the bytes we get
-                # We need to put the actual count in a container so our
-                # nested function can modify it without creating its own
-                # local with the same name.
-                size = [0]
-                def count(l: int) -> None:
-                    size[0] += l
-                counter = WriteWatchingStream(writable)
-                counter.onWrite(count)
-
-                # Do the download
-                shutil.copyfileobj(readable, counter)
-                return size[0], False
+            return cast(IO[bytes], closing(urlopen(url.geturl())))
         except HTTPError as e:
             if e.code == 404:
                 # Translate into a FileNotFoundError for detecting

toil/jobStores/aws/jobStore.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import hashlib
 import itertools
-import json
 import logging
 import os
 import pickle
@@ -21,12 +20,10 @@ import re
 import reprlib
 import stat
 import time
-import urllib.error
-import urllib.request
 import uuid
 from contextlib import contextmanager
 from io import BytesIO
-from typing import List, Optional
+from typing import List, Optional, IO
 from urllib.parse import ParseResult, parse_qs, urlencode, urlsplit, urlunsplit
 
 import boto.s3.connection
@@ -35,7 +32,6 @@ from boto.exception import SDBResponseError
 from botocore.exceptions import ClientError
 
 import toil.lib.encryption as encryption
-from toil.lib.aws import build_tag_dict_from_env
 from toil.fileStores import FileID
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
                                              ConcurrentFileModificationException,
@@ -56,6 +52,7 @@ from toil.jobStores.aws.utils import (SDBHelper,
 from toil.jobStores.utils import (ReadablePipe,
                                   ReadableTransformingPipe,
                                   WritablePipe)
+from toil.lib.aws import build_tag_dict_from_env
 from toil.lib.aws.session import establish_boto3_session
 from toil.lib.aws.utils import (create_s3_bucket,
                                 enable_public_objects,
@@ -450,7 +447,6 @@ class AWSJobStore(AbstractJobStore):
         except ServerSideCopyProhibitedError:
             # AWS refuses to do this copy for us
             logger.warning("Falling back to copying via the local machine. This could get expensive!")
-            pass
 
         # copy if exception
         return super()._import_file(otherCls, uri, shared_file_name=shared_file_name)
@@ -465,12 +461,21 @@ class AWSJobStore(AbstractJobStore):
         except ServerSideCopyProhibitedError:
             # AWS refuses to do this copy for us
             logger.warning("Falling back to copying via the local machine. This could get expensive!")
-            pass
         else:
             super()._default_export_file(otherCls, file_id, uri)
 
     @classmethod
-    def get_size(cls, url):
+    def _url_exists(cls, url: ParseResult) -> bool:
+        try:
+            get_object_for_url(url, existing=True)
+            return True
+        except FileNotFoundError:
+            # Not a file
+            # Might be a directory.
+            return cls._get_is_directory(url)
+
+    @classmethod
+    def _get_size(cls, url):
         return get_object_for_url(url, existing=True).content_length
 
     @classmethod
@@ -482,6 +487,15 @@ class AWSJobStore(AbstractJobStore):
             False  # executable bit is always False
         )
 
+    @classmethod
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
+        src_obj = get_object_for_url(url, existing=True)
+        response = src_obj.get()
+        # We should get back a response with a stream in 'Body'
+        if 'Body' not in response:
+            raise RuntimeError(f"Could not fetch body stream for {url}")
+        return response['Body']
+
     @classmethod
     def _write_to_url(cls, readable, url, executable=False):
         dstObj = get_object_for_url(url)
@@ -757,7 +771,7 @@ class AWSJobStore(AbstractJobStore):
                                 bucket_tagging.put(Tagging={'TagSet': flat_tags})
 
                             # Configure bucket so that we can make objects in
-                            # it public, which was the historical default. 
+                            # it public, which was the historical default.
                             enable_public_objects(bucket_name)
                         elif block:
                             raise

toil/jobStores/aws/utils.py

@@ -17,13 +17,12 @@ import logging
 import os
 import types
 from ssl import SSLError
-from typing import Optional, cast
+from typing import Optional, cast, TYPE_CHECKING
 
 from boto3.s3.transfer import TransferConfig
-from boto.exception import BotoServerError, S3ResponseError, SDBResponseError
+from boto.exception import SDBResponseError
 from botocore.client import Config
 from botocore.exceptions import ClientError
-from mypy_boto3_s3 import S3Client, S3ServiceResource
 
 from toil.lib.aws import session
 from toil.lib.aws.utils import connection_reset, get_bucket_region
@@ -36,6 +35,8 @@ from toil.lib.retry import (DEFAULT_DELAYS,
                             get_error_status,
                             old_retry,
                             retry)
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client, S3ServiceResource
 
 logger = logging.getLogger(__name__)
 
@@ -285,13 +286,12 @@ class ServerSideCopyProhibitedError(RuntimeError):
     Raised when AWS refuses to perform a server-side copy between S3 keys, and
     insists that you pay to download and upload the data yourself instead.
     """
-    pass
 
 @retry(errors=[ErrorCondition(
     error=ClientError,
     error_codes=[404, 500, 502, 503, 504]
 )])
-def copyKeyMultipart(resource: S3ServiceResource,
+def copyKeyMultipart(resource: "S3ServiceResource",
                      srcBucketName: str,
                      srcKeyName: str,
                      srcKeyVersion: str,
@@ -347,7 +347,7 @@ def copyKeyMultipart(resource: S3ServiceResource,
     # not wherever the bucket virtual hostnames go.
     source_region = get_bucket_region(srcBucketName)
     source_client = cast(
-        S3Client,
+        "S3Client",
         session.client(
             's3',
             region_name=source_region,