google-cloud-storage 3.2.0__tar.gz → 3.3.1__tar.gz

{google_cloud_storage-3.2.0/google_cloud_storage.egg-info → google_cloud_storage-3.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-storage
-Version: 3.2.0
+Version: 3.3.1
 Summary: Google Cloud Storage API client library
 Home-page: https://github.com/googleapis/python-storage
 Author: Google LLC

{google_cloud_storage-3.2.0 → google_cloud_storage-3.3.1}/google/cloud/storage/_media/_upload.py

@@ -411,7 +411,12 @@ class ResumableUpload(UploadBase):
     """
 
     def __init__(
-        self, upload_url, chunk_size, checksum="auto", headers=None, retry=DEFAULT_RETRY
+        self,
+        upload_url,
+        chunk_size,
+        checksum="auto",
+        headers=None,
+        retry=DEFAULT_RETRY,
     ):
         super(ResumableUpload, self).__init__(upload_url, headers=headers, retry=retry)
         if chunk_size % UPLOAD_CHUNK_SIZE != 0:
@@ -472,7 +477,12 @@ class ResumableUpload(UploadBase):
         return self._total_bytes
 
     def _prepare_initiate_request(
-        self, stream, metadata, content_type, total_bytes=None, stream_final=True
+        self,
+        stream,
+        metadata,
+        content_type,
+        total_bytes=None,
+        stream_final=True,
     ):
         """Prepare the contents of HTTP request to initiate upload.
 
@@ -955,7 +965,12 @@ class XMLMPUContainer(UploadBase):
     """
 
     def __init__(
-        self, upload_url, filename, headers=None, upload_id=None, retry=DEFAULT_RETRY
+        self,
+        upload_url,
+        filename,
+        headers=None,
+        upload_id=None,
+        retry=DEFAULT_RETRY,
     ):
         super().__init__(upload_url, headers=headers, retry=retry)
         self._filename = filename
@@ -1372,6 +1387,29 @@ class XMLMPUPart(UploadBase):
 
         .. _sans-I/O: https://sans-io.readthedocs.io/
         """
+        # Data corruption errors shouldn't be considered as invalid responses,
+        # So we handle them earlier than call to `_helpers.require_status_code`.
+        # If the response is 400, we check for data corruption errors.
+        if response.status_code == 400:
+            root = ElementTree.fromstring(response.text)
+            error_code = root.find("Code").text
+            error_message = root.find("Message").text
+            error_details = root.find("Details").text
+            if error_code in ["InvalidDigest", "BadDigest", "CrcMismatch"]:
+                raise DataCorruption(
+                    response,
+                    (
+                        "Checksum mismatch: checksum calculated by client and"
+                        " server did not match. Error code: {error_code},"
+                        " Error message: {error_message},"
+                        " Error details: {error_details}"
+                    ).format(
+                        error_code=error_code,
+                        error_message=error_message,
+                        error_details=error_details,
+                    ),
+                )
+
         _helpers.require_status_code(
             response,
             (http.client.OK,),

{google_cloud_storage-3.2.0 → google_cloud_storage-3.3.1}/google/cloud/storage/_media/requests/upload.py

@@ -21,6 +21,7 @@ uploads that contain both metadata and a small file as payload.
 
 from google.cloud.storage._media import _upload
 from google.cloud.storage._media.requests import _request_helpers
+from google.cloud.storage._media import _helpers
 
 
 class SimpleUpload(_request_helpers.RequestsMixin, _upload.SimpleUpload):
@@ -757,6 +758,14 @@ class XMLMPUPart(_request_helpers.RequestsMixin, _upload.XMLMPUPart):
             ~requests.Response: The HTTP response returned by ``transport``.
         """
         method, url, payload, headers = self._prepare_upload_request()
+        if self._checksum_object is not None:
+            checksum_digest_in_base64 = _helpers.prepare_checksum_digest(
+                self._checksum_object.digest()
+            )
+            if self._checksum_type == "crc32c":
+                headers["X-Goog-Hash"] = f"crc32c={checksum_digest_in_base64}"
+            elif self._checksum_type == "md5":
+                headers["X-Goog-Hash"] = f"md5={checksum_digest_in_base64}"
 
         # Wrap the request business logic in a function to be retried.
         def retriable_request():

{google_cloud_storage-3.2.0 → google_cloud_storage-3.3.1}/google/cloud/storage/blob.py

@@ -14,8 +14,7 @@
 
 # pylint: disable=too-many-lines
 
-"""Create / interact with Google Cloud Storage blobs.
-"""
+"""Create / interact with Google Cloud Storage blobs."""
 
 import base64
 import copy
@@ -142,8 +141,8 @@ _GS_URL_REGEX_PATTERN = re.compile(
     r"(?P<scheme>gs)://(?P<bucket_name>[a-z0-9_.-]+)/(?P<object_name>.+)"
 )
 
-_DEFAULT_CHUNKSIZE = 104857600  # 1024 * 1024 B * 100 = 100 MB
-_MAX_MULTIPART_SIZE = 8388608  # 8 MB
+_DEFAULT_CHUNKSIZE = 104857600  # 1024 * 1024 B * 100 = 100 MiB
+_MAX_MULTIPART_SIZE = 8388608  # 8 MiB
 
 _logger = logging.getLogger(__name__)
 
@@ -181,6 +180,14 @@ class Blob(_PropertyMixin):
     :type generation: long
     :param generation:
         (Optional) If present, selects a specific revision of this object.
+
+    :type crc32c_checksum: str
+    :param crc32c_checksum:
+            (Optional) If set, the CRC32C checksum of the blob's content.
+            CRC32c checksum, as described in RFC 4960, Appendix B; encoded using
+            base64 in big-endian byte order. See
+            Apenndix B: https://datatracker.ietf.org/doc/html/rfc4960#appendix-B
+            base64: https://datatracker.ietf.org/doc/html/rfc4648#section-4
     """
 
     _chunk_size = None  # Default value for each instance.
@@ -214,6 +221,7 @@ class Blob(_PropertyMixin):
         encryption_key=None,
         kms_key_name=None,
         generation=None,
+        crc32c_checksum=None,
     ):
         """
         property :attr:`name`
@@ -237,6 +245,9 @@ class Blob(_PropertyMixin):
         if generation is not None:
             self._properties["generation"] = generation
 
+        if crc32c_checksum is not None:
+            self._properties["crc32c"] = crc32c_checksum
+
     @property
     def bucket(self):
         """Bucket which contains the object.
@@ -1643,7 +1654,9 @@ class Blob(_PropertyMixin):
         :raises: :class:`google.cloud.exceptions.NotFound`
         """
         warnings.warn(
-            _DOWNLOAD_AS_STRING_DEPRECATED, PendingDeprecationWarning, stacklevel=2
+            _DOWNLOAD_AS_STRING_DEPRECATED,
+            PendingDeprecationWarning,
+            stacklevel=2,
         )
         with create_trace_span(name="Storage.Blob.downloadAsString"):
             return self.download_as_bytes(
@@ -1999,12 +2012,18 @@ class Blob(_PropertyMixin):
         transport = self._get_transport(client)
         if "metadata" in self._properties and "metadata" not in self._changes:
             self._changes.add("metadata")
+
         info = self._get_upload_arguments(client, content_type, command=command)
         headers, object_metadata, content_type = info
 
+        if "crc32c" in self._properties:
+            object_metadata["crc32c"] = self._properties["crc32c"]
+
         hostname = _get_host_name(client._connection)
         base_url = _MULTIPART_URL_TEMPLATE.format(
-            hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
+            hostname=hostname,
+            bucket_path=self.bucket.path,
+            api_version=_API_VERSION,
         )
         name_value_pairs = []
 
@@ -2195,9 +2214,14 @@ class Blob(_PropertyMixin):
         if extra_headers is not None:
             headers.update(extra_headers)
 
+        if "crc32c" in self._properties:
+            object_metadata["crc32c"] = self._properties["crc32c"]
+
         hostname = _get_host_name(client._connection)
         base_url = _RESUMABLE_URL_TEMPLATE.format(
-            hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
+            hostname=hostname,
+            bucket_path=self.bucket.path,
+            api_version=_API_VERSION,
         )
         name_value_pairs = []
 
@@ -2234,7 +2258,11 @@ class Blob(_PropertyMixin):
 
         upload_url = _add_query_parameters(base_url, name_value_pairs)
         upload = ResumableUpload(
-            upload_url, chunk_size, headers=headers, checksum=checksum, retry=retry
+            upload_url,
+            chunk_size,
+            headers=headers,
+            checksum=checksum,
+            retry=retry,
         )
 
         upload.initiate(
@@ -3426,7 +3454,11 @@ class Blob(_PropertyMixin):
             return Policy.from_api_repr(info)
 
     def test_iam_permissions(
-        self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
+        self,
+        permissions,
+        client=None,
+        timeout=_DEFAULT_TIMEOUT,
+        retry=DEFAULT_RETRY,
     ):
         """API call:  test permissions
 
@@ -3693,7 +3725,10 @@ class Blob(_PropertyMixin):
 
             source_objects = []
             for source, source_generation in zip(sources, if_source_generation_match):
-                source_object = {"name": source.name, "generation": source.generation}
+                source_object = {
+                    "name": source.name,
+                    "generation": source.generation,
+                }
 
                 preconditions = {}
                 if source_generation is not None:
@@ -4154,7 +4189,10 @@ class Blob(_PropertyMixin):
                         "encoding, errors and newline arguments are for text mode only"
                     )
                 return BlobWriter(
-                    self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs
+                    self,
+                    chunk_size=chunk_size,
+                    ignore_flush=ignore_flush,
+                    **kwargs,
                 )
             elif mode in ("r", "rt"):
                 if ignore_flush:

{google_cloud_storage-3.2.0 → google_cloud_storage-3.3.1}/google/cloud/storage/bucket.py

@@ -56,6 +56,7 @@ from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_INHERITED
 from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS
 from google.cloud.storage.constants import REGION_LOCATION_TYPE
 from google.cloud.storage.constants import STANDARD_STORAGE_CLASS
+from google.cloud.storage.ip_filter import IPFilter
 from google.cloud.storage.notification import BucketNotification
 from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
 from google.cloud.storage.retry import DEFAULT_RETRY
@@ -88,6 +89,7 @@ _LOCATION_SETTER_MESSAGE = (
 _FROM_STRING_MESSAGE = (
     "Bucket.from_string() is deprecated. " "Use Bucket.from_uri() instead."
 )
+_IP_FILTER_PROPERTY = "ipFilter"
 
 
 def _blobs_page_start(iterator, page, response):
@@ -384,7 +386,10 @@ class LifecycleRuleSetStorageClass(dict):
     def __init__(self, storage_class, **kw):
         conditions = LifecycleRuleConditions(**kw)
         rule = {
-            "action": {"type": "SetStorageClass", "storageClass": storage_class},
+            "action": {
+                "type": "SetStorageClass",
+                "storageClass": storage_class,
+            },
             "condition": dict(conditions),
         }
         super().__init__(rule)
@@ -844,6 +849,7 @@ class Bucket(_PropertyMixin):
         encryption_key=None,
         kms_key_name=None,
         generation=None,
+        crc32c_checksum=None,
     ):
         """Factory constructor for blob object.
 
@@ -871,6 +877,14 @@ class Bucket(_PropertyMixin):
         :param generation: (Optional) If present, selects a specific revision of
                            this object.
 
+        :type crc32c_checksum: str
+        :param crc32c_checksum:
+            (Optional) If set, the CRC32C checksum of the blob's content.
+            CRC32c checksum, as described in RFC 4960, Appendix B; encoded using
+            base64 in big-endian byte order. See
+            Apenndix B: https://datatracker.ietf.org/doc/html/rfc4960#appendix-B
+            base64: https://datatracker.ietf.org/doc/html/rfc4648#section-4
+
         :rtype: :class:`google.cloud.storage.blob.Blob`
         :returns: The blob object created.
         """
@@ -881,6 +895,7 @@ class Bucket(_PropertyMixin):
             encryption_key=encryption_key,
             kms_key_name=kms_key_name,
             generation=generation,
+            crc32c_checksum=crc32c_checksum,
         )
 
     def notification(
@@ -2255,10 +2270,7 @@ class Bucket(_PropertyMixin):
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
     ):
-        """Move a blob to a new name within a single HNS bucket.
-
-        *This feature is currently only supported for HNS (Heirarchical
-        Namespace) buckets.*
+        """Move a blob to a new name atomically.
 
         If :attr:`user_project` is set on the bucket, bills the API request to that project.
 
@@ -3254,7 +3266,10 @@ class Bucket(_PropertyMixin):
         :type not_found_page: str
         :param not_found_page: The file to use when a page isn't found.
         """
-        data = {"mainPageSuffix": main_page_suffix, "notFoundPage": not_found_page}
+        data = {
+            "mainPageSuffix": main_page_suffix,
+            "notFoundPage": not_found_page,
+        }
         self._patch_property("website", data)
 
     def disable_website(self):
@@ -3386,7 +3401,11 @@ class Bucket(_PropertyMixin):
             return Policy.from_api_repr(info)
 
     def test_iam_permissions(
-        self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY
+        self,
+        permissions,
+        client=None,
+        timeout=_DEFAULT_TIMEOUT,
+        retry=DEFAULT_RETRY,
     ):
         """API call:  test permissions
 
@@ -3890,6 +3909,59 @@ class Bucket(_PropertyMixin):
             query_parameters=query_parameters,
         )
 
+    @property
+    def ip_filter(self):
+        """Retrieve or set the IP Filter configuration for this bucket.
+
+        See https://cloud.google.com/storage/docs/ip-filtering-overview and
+        https://cloud.google.com/storage/docs/json_api/v1/buckets#ipFilter
+
+        .. note::
+            The getter for this property returns an
+            :class:`~google.cloud.storage.ip_filter.IPFilter` object, which is a
+            structured representation of the bucket's IP filter configuration.
+            Modifying the returned object has no effect. To update the bucket's
+            IP filter, create and assign a new ``IPFilter`` object to this
+            property and then call
+            :meth:`~google.cloud.storage.bucket.Bucket.patch`.
+
+            .. code-block:: python
+
+                from google.cloud.storage.ip_filter import (
+                    IPFilter,
+                    PublicNetworkSource,
+                )
+
+                ip_filter = IPFilter()
+                ip_filter.mode = "Enabled"
+                ip_filter.public_network_source = PublicNetworkSource(
+                    allowed_ip_cidr_ranges=["203.0.113.5/32"]
+                )
+                bucket.ip_filter = ip_filter
+                bucket.patch()
+
+        :setter: Set the IP Filter configuration for this bucket.
+        :getter: Gets the IP Filter configuration for this bucket.
+
+        :rtype: :class:`~google.cloud.storage.ip_filter.IPFilter` or ``NoneType``
+        :returns:
+            An ``IPFilter`` object representing the configuration, or ``None``
+            if no filter is configured.
+        """
+        resource = self._properties.get(_IP_FILTER_PROPERTY)
+        if resource:
+            return IPFilter._from_api_resource(resource)
+        return None
+
+    @ip_filter.setter
+    def ip_filter(self, value):
+        if value is None:
+            self._patch_property(_IP_FILTER_PROPERTY, None)
+        elif isinstance(value, IPFilter):
+            self._patch_property(_IP_FILTER_PROPERTY, value._to_api_resource())
+        else:
+            self._patch_property(_IP_FILTER_PROPERTY, value)
+
 
 class SoftDeletePolicy(dict):
     """Map a bucket's soft delete policy.

google_cloud_storage-3.3.1/google/cloud/storage/ip_filter.py

@@ -0,0 +1,143 @@
+# Copyright 2014 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IP Filter configuration for Google Cloud Storage Buckets."""
+
+from typing import Dict, Any, Optional, List
+
+_MODE = "mode"
+_PUBLIC_NETWORK_SOURCE = "publicNetworkSource"
+_VPC_NETWORK_SOURCES = "vpcNetworkSources"
+_ALLOWED_IP_CIDR_RANGES = "allowedIpCidrRanges"
+_NETWORK = "network"
+_ALLOW_ALL_SERVICE_AGENT_ACCESS = "allowAllServiceAgentAccess"
+_ALLOW_CROSS_ORG_VPCS = "allowCrossOrgVpcs"
+
+
+class PublicNetworkSource:
+    """Represents a public network source for a GCS Bucket IP Filter.
+
+    :type allowed_ip_cidr_ranges: list(str) or None
+    :param allowed_ip_cidr_ranges: A list of public IPv4 or IPv6 ranges in
+                                   CIDR notation that are allowed to access
+                                   the bucket.
+    """
+
+    def __init__(self, allowed_ip_cidr_ranges: Optional[List[str]] = None):
+        self.allowed_ip_cidr_ranges = allowed_ip_cidr_ranges or []
+
+    def _to_api_resource(self) -> Dict[str, Any]:
+        """Serializes this object to a dictionary for API requests."""
+        return {_ALLOWED_IP_CIDR_RANGES: self.allowed_ip_cidr_ranges}
+
+
+class VpcNetworkSource:
+    """Represents a VPC network source for a GCS Bucket IP Filter.
+
+    :type network: str
+    :param network: The resource name of the VPC network.
+
+    :type allowed_ip_cidr_ranges: list(str) or None
+    :param allowed_ip_cidr_ranges: A list of IPv4 or IPv6 ranges in CIDR
+                                   notation allowed to access the bucket
+                                   from this VPC.
+    """
+
+    def __init__(
+        self, network: str, allowed_ip_cidr_ranges: Optional[List[str]] = None
+    ):
+        self.network = network
+        self.allowed_ip_cidr_ranges = allowed_ip_cidr_ranges or []
+
+    def _to_api_resource(self) -> Dict[str, Any]:
+        """Serializes this object to a dictionary for API requests."""
+        return {
+            _NETWORK: self.network,
+            _ALLOWED_IP_CIDR_RANGES: self.allowed_ip_cidr_ranges,
+        }
+
+
+class IPFilter:
+    """Represents a GCS Bucket IP Filter configuration.
+
+    This class is a helper for constructing the IP Filter dictionary to be
+    assigned to a bucket's ``ip_filter`` property.
+    """
+
+    """
+    Attributes:
+        mode (str): Required. The mode of the IP filter. Can be "Enabled" or "Disabled".
+        allow_all_service_agent_access (bool): Required. If True, allows Google
+            Cloud service agents to bypass the IP filter.
+        public_network_source (PublicNetworkSource): (Optional) The configuration
+            for requests from the public internet.
+        vpc_network_sources (list(VpcNetworkSource)): (Optional) A list of
+            configurations for requests from VPC networks.
+        allow_cross_org_vpcs (bool): (Optional) If True, allows VPCs from
+            other organizations to be used in the configuration.
+    """
+
+    def __init__(self):
+        self.mode: Optional[str] = None
+        self.public_network_source: Optional[PublicNetworkSource] = None
+        self.vpc_network_sources: List[VpcNetworkSource] = []
+        self.allow_all_service_agent_access: Optional[bool] = None
+        self.allow_cross_org_vpcs: Optional[bool] = None
+
+    @classmethod
+    def _from_api_resource(cls, resource: Dict[str, Any]) -> "IPFilter":
+        """Factory: creates an IPFilter instance from a server response."""
+        ip_filter = cls()
+        ip_filter.mode = resource.get(_MODE)
+        ip_filter.allow_all_service_agent_access = resource.get(
+            _ALLOW_ALL_SERVICE_AGENT_ACCESS, None
+        )
+
+        public_network_source_data = resource.get(_PUBLIC_NETWORK_SOURCE, None)
+        if public_network_source_data:
+            ip_filter.public_network_source = PublicNetworkSource(
+                allowed_ip_cidr_ranges=public_network_source_data.get(
+                    _ALLOWED_IP_CIDR_RANGES, []
+                )
+            )
+
+        vns_res_list = resource.get(_VPC_NETWORK_SOURCES, [])
+        ip_filter.vpc_network_sources = [
+            VpcNetworkSource(
+                network=vns.get(_NETWORK),
+                allowed_ip_cidr_ranges=vns.get(_ALLOWED_IP_CIDR_RANGES, []),
+            )
+            for vns in vns_res_list
+        ]
+        ip_filter.allow_cross_org_vpcs = resource.get(_ALLOW_CROSS_ORG_VPCS, None)
+        return ip_filter
+
+    def _to_api_resource(self) -> Dict[str, Any]:
+        """Serializes this object to a dictionary for API requests."""
+        resource = {
+            _MODE: self.mode,
+            _ALLOW_ALL_SERVICE_AGENT_ACCESS: self.allow_all_service_agent_access,
+        }
+
+        if self.public_network_source:
+            resource[
+                _PUBLIC_NETWORK_SOURCE
+            ] = self.public_network_source._to_api_resource()
+        if self.vpc_network_sources is not None:
+            resource[_VPC_NETWORK_SOURCES] = [
+                vns._to_api_resource() for vns in self.vpc_network_sources
+            ]
+        if self.allow_cross_org_vpcs is not None:
+            resource[_ALLOW_CROSS_ORG_VPCS] = self.allow_cross_org_vpcs
+        return resource

{google_cloud_storage-3.2.0 → google_cloud_storage-3.3.1}/google/cloud/storage/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.2.0"
+__version__ = "3.3.1"