toil 8.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/jobStores/aws/jobStore.py

@@ -637,30 +637,61 @@ class AWSJobStore(AbstractJobStore):
         else:
             super()._default_export_file(otherCls, file_id, uri)
 
+    ###
+    # URL access implementation
+    ###
+
+    # URL access methods aren't used by the rest of the job store methods.
+
     @classmethod
     def _url_exists(cls, url: ParseResult) -> bool:
         try:
-            get_object_for_url(url, existing=True)
+            try:
+                get_object_for_url(url, existing=True, anonymous=True)
+            except PermissionError:
+                # If we can't look anonymously, log in
+                get_object_for_url(url, existing=True)
             return True
         except FileNotFoundError:
             # Not a file
-            # Might be a directory.
+            # Might be a directory. Or we might not have access to know.
+            # See if it's a directory.
             return cls._get_is_directory(url)
 
     @classmethod
     def _get_size(cls, url: ParseResult) -> int:
-        return get_object_for_url(url, existing=True).content_length
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+        except PermissionError:
+            src_obj = get_object_for_url(url, existing=True)
+        return src_obj.content_length
 
     @classmethod
     def _read_from_url(cls, url: ParseResult, writable):
-        srcObj = get_object_for_url(url, existing=True)
-        srcObj.download_fileobj(writable)
-        return (srcObj.content_length, False)  # executable bit is always False
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            src_obj.download_fileobj(writable)
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                src_obj.download_fileobj(writable)
+            else:
+                raise
+        return (src_obj.content_length, False)  # executable bit is always False
 
     @classmethod
     def _open_url(cls, url: ParseResult) -> IO[bytes]:
-        src_obj = get_object_for_url(url, existing=True)
-        response = src_obj.get()
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            response = src_obj.get()
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                response = src_obj.get()
+            else:
+                raise
         # We should get back a response with a stream in 'Body'
         if "Body" not in response:
             raise RuntimeError(f"Could not fetch body stream for {url}")
@@ -670,6 +701,7 @@ class AWSJobStore(AbstractJobStore):
     def _write_to_url(
         cls, readable, url: ParseResult, executable: bool = False
     ) -> None:
+        # Don't try to do anonympus writes.
         dstObj = get_object_for_url(url)
 
         logger.debug("Uploading %s", dstObj.key)
@@ -684,13 +716,17 @@ class AWSJobStore(AbstractJobStore):
 
     @classmethod
     def _list_url(cls, url: ParseResult) -> list[str]:
-        return list_objects_for_url(url)
+        try:
+            return list_objects_for_url(url, anonymous=True)
+        except PermissionError:
+            return list_objects_for_url(url)
+        
 
     @classmethod
     def _get_is_directory(cls, url: ParseResult) -> bool:
         # We consider it a directory if anything is in it.
         # TODO: Can we just get the first item and not the whole list?
-        return len(list_objects_for_url(url)) > 0
+        return len(cls._list_url(url)) > 0
 
     @classmethod
     def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:

toil/lib/aws/session.py

@@ -35,6 +35,9 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+# You can pass config=ANONYMOUS_CONFIG to make anonymous S3 accesses
+ANONYMOUS_CONFIG = Config(signature_version=botocore.UNSIGNED)
+
 # A note on thread safety:
 #
 # Boto3 Session: Not thread safe, 1 per thread is required.
@@ -148,6 +151,7 @@ class AWSConnectionManager:
         region: Optional[str],
         service_name: Literal["s3"],
         endpoint_url: Optional[str] = None,
+        config: Optional[Config] = None,
     ) -> "S3ServiceResource": ...
     @overload
     def resource(
@@ -155,6 +159,7 @@ class AWSConnectionManager:
         region: Optional[str],
         service_name: Literal["iam"],
         endpoint_url: Optional[str] = None,
+        config: Optional[Config] = None,
     ) -> "IAMServiceResource": ...
     @overload
     def resource(
@@ -162,6 +167,7 @@ class AWSConnectionManager:
         region: Optional[str],
         service_name: Literal["ec2"],
         endpoint_url: Optional[str] = None,
+        config: Optional[Config] = None,
     ) -> "EC2ServiceResource": ...
 
     def resource(
@@ -169,6 +175,7 @@ class AWSConnectionManager:
         region: Optional[str],
         service_name: str,
         endpoint_url: Optional[str] = None,
+        config: Optional[Config] = None,
     ) -> boto3.resources.base.ServiceResource:
         """
         Get the Boto3 Resource to use with the given service (like 'ec2') in the given region.
@@ -188,10 +195,10 @@ class AWSConnectionManager:
                     # The Boto3 stubs are missing an overload for `resource` that takes
                     # a non-literal string. See
                     # <https://github.com/vemel/mypy_boto3_builder/issues/121#issuecomment-1011322636>
-                    storage.item = self.session(region).resource(service_name, endpoint_url=endpoint_url)  # type: ignore
+                    storage.item = self.session(region).resource(service_name, endpoint_url=endpoint_url, config=config)  # type: ignore
                 else:
                     # We might not be able to pass None to Boto3 and have it be the same as no argument.
-                    storage.item = self.session(region).resource(service_name)  # type: ignore
+                    storage.item = self.session(region).resource(service_name, config=config)  # type: ignore
 
         return cast(boto3.resources.base.ServiceResource, storage.item)
 
@@ -369,18 +376,21 @@ def resource(
     service_name: Literal["s3"],
     region_name: Optional[str] = None,
     endpoint_url: Optional[str] = None,
+    config: Optional[Config] = None,
 ) -> "S3ServiceResource": ...
 @overload
 def resource(
     service_name: Literal["iam"],
     region_name: Optional[str] = None,
     endpoint_url: Optional[str] = None,
+    config: Optional[Config] = None,
 ) -> "IAMServiceResource": ...
 @overload
 def resource(
     service_name: Literal["ec2"],
     region_name: Optional[str] = None,
     endpoint_url: Optional[str] = None,
+    config: Optional[Config] = None,
 ) -> "EC2ServiceResource": ...
 
 
@@ -388,6 +398,7 @@ def resource(
     service_name: Literal["s3", "iam", "ec2"],
     region_name: Optional[str] = None,
     endpoint_url: Optional[str] = None,
+    config: Optional[Config] = None,
 ) -> boto3.resources.base.ServiceResource:
     """
     Get a Boto 3 resource for a particular AWS service, usable by the current thread.
@@ -397,5 +408,5 @@ def resource(
 
     # Just use a global version of the manager. Note that we change the argument order!
     return _global_manager.resource(
-        region_name, service_name, endpoint_url=endpoint_url
+        region_name, service_name, endpoint_url=endpoint_url, config=config
     )

toil/lib/aws/utils.py

@@ -19,8 +19,10 @@ from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Optional, cast
 from urllib.parse import ParseResult
 
+# To import toil.lib.aws.session, the AWS libraries must be installed 
 from toil.lib.aws import AWSRegionName, AWSServerErrors, session
 from toil.lib.conversions import strtobool
+from toil.lib.memoize import memoize
 from toil.lib.misc import printq
 from toil.lib.retry import (
     DEFAULT_DELAYS,
@@ -37,12 +39,7 @@ if TYPE_CHECKING:
     from mypy_boto3_s3.service_resource import Object as S3Object
     from mypy_boto3_sdb.type_defs import AttributeTypeDef
 
-try:
-    from botocore.exceptions import ClientError, EndpointConnectionError
-except ImportError:
-    ClientError = None  # type: ignore
-    EndpointConnectionError = None  # type: ignore
-    # AWS/boto extra is not installed
+from botocore.exceptions import ClientError, EndpointConnectionError
 
 logger = logging.getLogger(__name__)
 
@@ -232,6 +229,7 @@ def get_bucket_region(
     bucket_name: str,
     endpoint_url: Optional[str] = None,
     only_strategies: Optional[set[int]] = None,
+    anonymous: Optional[bool] = None
 ) -> str:
     """
     Get the AWS region name associated with the given S3 bucket, or raise NoBucketLocationError.
@@ -241,9 +239,13 @@ def get_bucket_region(
     Takes an optional S3 API URL override.
 
     :param only_strategies: For testing, use only strategies with 1-based numbers in this set.
+
+    :raises NoBucketLocationError: if the bucket's region cannot be determined
+        (possibly due to lack of permissions).
     """
 
-    s3_client = session.client("s3", endpoint_url=endpoint_url)
+    config = session.ANONYMOUS_CONFIG if anonymous else None
+    s3_client = session.client("s3", endpoint_url=endpoint_url, config=config)
 
     def attempt_get_bucket_location() -> Optional[str]:
         """
@@ -267,7 +269,7 @@ def get_bucket_region(
         # It could also be because AWS open data buckets (which we tend to
         # encounter this problem for) tend to actually themselves be in
         # us-east-1.
-        backup_s3_client = session.client("s3", region_name="us-east-1")
+        backup_s3_client = session.client("s3", region_name="us-east-1", config=config)
         return backup_s3_client.get_bucket_location(Bucket=bucket_name).get(
             "LocationConstraint", None
         )
@@ -337,6 +339,30 @@ def get_bucket_region(
         "Could not get bucket location: " + "\n".join(error_messages)
     ) from last_error
 
+@memoize
+def get_bucket_region_if_available(
+    bucket_name: str,
+    endpoint_url: Optional[str] = None,
+    only_strategies: Optional[set[int]] = None,
+    anonymous: Optional[bool] = None
+) -> Optional[str]:
+    """
+    Get the AWS region name associated with the given S3 bucket, or return None.
+
+    Caches results, so may not return the location for a bucket that has been
+    created but was previously observed to be nonexistent.
+
+    :param only_strategies: For testing, use only strategies with 1-based numbers in this set.
+    """
+
+    try:
+        return get_bucket_region(bucket_name, endpoint_url, only_strategies, anonymous)
+    except Exception as e:
+        if isinstance(e, NoBucketLocationError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+            # We can't know
+            return None
+        else:
+            raise
 
 def region_to_bucket_location(region: str) -> str:
     return "" if region == "us-east-1" else region
@@ -346,7 +372,7 @@ def bucket_location_to_region(location: Optional[str]) -> str:
     return "us-east-1" if location == "" or location is None else location
 
 
-def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3Object":
+def get_object_for_url(url: ParseResult, existing: Optional[bool] = None, anonymous: Optional[bool] = None) -> "S3Object":
     """
     Extracts a key (object) from a given parsed s3:// URL.
 
@@ -354,6 +380,10 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3
 
     :param bool existing: If True, key is expected to exist. If False, key is expected not to
             exists and it will be created. If None, the key will be created if it doesn't exist.
+
+    :raises FileNotFoundError: when existing is True and the object does not exist.
+    :raises RuntimeError: when existing is False but the object exists.
+    :raises PermissionError: when we are not authorized to look at the object.
     """
 
     key_name = url.path[1:]
@@ -372,17 +402,19 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3
     # TODO: OrdinaryCallingFormat equivalent in boto3?
     # if botoargs:
     #     botoargs['calling_format'] = boto.s3.connection.OrdinaryCallingFormat()
-
-    try:
-        # Get the bucket's region to avoid a redirect per request
-        region = get_bucket_region(bucket_name, endpoint_url=endpoint_url)
-        s3 = session.resource("s3", region_name=region, endpoint_url=endpoint_url)
-    except NoBucketLocationError as e:
-        # Probably don't have permission.
-        # TODO: check if it is that
-        logger.debug("Couldn't get bucket location: %s", e)
+    
+    config = session.ANONYMOUS_CONFIG if anonymous else None
+    # Get the bucket's region to avoid a redirect per request.
+    # Cache the result
+    region = get_bucket_region_if_available(bucket_name, endpoint_url=endpoint_url, anonymous=anonymous)
+    if region is not None:
+        s3 = session.resource("s3", region_name=region, endpoint_url=endpoint_url, config=config)
+    else:
+        # We can't get the bucket location, perhaps because we don't have
+        # permission to do that.
+        logger.debug("Couldn't get bucket location")
         logger.debug("Fall back to not specifying location")
-        s3 = session.resource("s3", endpoint_url=endpoint_url)
+        s3 = session.resource("s3", endpoint_url=endpoint_url, config=config)
 
     obj = s3.Object(bucket_name, key_name)
     objExists = True
@@ -392,6 +424,10 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3
     except ClientError as e:
         if get_error_status(e) == 404:
             objExists = False
+        elif get_error_status(e) == 403:
+            raise PermissionError(
+                f"Key '{key_name}' is not accessible in bucket '{bucket_name}'."
+            ) from e
         else:
             raise
     if existing is True and not objExists:
@@ -402,16 +438,27 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3
         raise RuntimeError(f"Key '{key_name}' exists in bucket '{bucket_name}'.")
 
     if not objExists:
-        obj.put()  # write an empty file
+        try:
+            obj.put()  # write an empty file
+        except ClientError as e:
+            if get_error_status(e) == 403:
+                raise PermissionError(
+                    f"Key '{key_name}' is not writable in bucket '{bucket_name}'."
+                ) from e
+            else:
+                raise
     return obj
 
 
 @retry(errors=[AWSServerErrors])
-def list_objects_for_url(url: ParseResult) -> list[str]:
+def list_objects_for_url(url: ParseResult, anonymous: Optional[bool] = None) -> list[str]:
     """
     Extracts a key (object) from a given parsed s3:// URL. The URL will be
     supplemented with a trailing slash if it is missing.
+
+    :raises PermissionError: when we are not authorized to do the list operation.
     """
+
     key_name = url.path[1:]
     bucket_name = url.netloc
 
@@ -430,23 +477,33 @@ def list_objects_for_url(url: ParseResult) -> list[str]:
         protocol = "http"
     if host:
         endpoint_url = f"{protocol}://{host}" + f":{port}" if port else ""
-
-    client = session.client("s3", endpoint_url=endpoint_url)
+    
+    config = session.ANONYMOUS_CONFIG if anonymous else None
+    client = session.client("s3", endpoint_url=endpoint_url, config=config)
 
     listing = []
+    
+    try:
+        paginator = client.get_paginator("list_objects_v2")
+        result = paginator.paginate(Bucket=bucket_name, Prefix=key_name, Delimiter="/")
+        for page in result:
+            if "CommonPrefixes" in page:
+                for prefix_item in page["CommonPrefixes"]:
+                    listing.append(prefix_item["Prefix"][len(key_name) :])
+            if "Contents" in page:
+                for content_item in page["Contents"]:
+                    if content_item["Key"] == key_name:
+                        # Ignore folder name itself
+                        continue
+                    listing.append(content_item["Key"][len(key_name) :])
+    except ClientError as e:
+        if get_error_status(e) == 403:
+            raise PermissionError(
+                f"Prefix '{key_name}' is not authorized to be listed in bucket '{bucket_name}'."
+            ) from e
+        else:
+            raise
 
-    paginator = client.get_paginator("list_objects_v2")
-    result = paginator.paginate(Bucket=bucket_name, Prefix=key_name, Delimiter="/")
-    for page in result:
-        if "CommonPrefixes" in page:
-            for prefix_item in page["CommonPrefixes"]:
-                listing.append(prefix_item["Prefix"][len(key_name) :])
-        if "Contents" in page:
-            for content_item in page["Contents"]:
-                if content_item["Key"] == key_name:
-                    # Ignore folder name itself
-                    continue
-                listing.append(content_item["Key"][len(key_name) :])
 
     logger.debug("Found in %s items: %s", url, listing)
     return listing