google-cloud-storage 2.18.2__tar.gz → 2.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_storage-2.18.2/google_cloud_storage.egg-info → google_cloud_storage-2.19.0}/PKG-INFO +1 -1
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_helpers.py +3 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_opentelemetry_tracing.py +7 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_signing.py +26 -8
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/blob.py +4 -1
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/bucket.py +63 -3
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/client.py +127 -11
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/transfer_manager.py +2 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/version.py +1 -1
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0/google_cloud_storage.egg-info}/PKG-INFO +1 -1
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/_helpers.py +4 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/conftest.py +83 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test__signing.py +82 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_client.py +75 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test__opentelemetry_tracing.py +9 -14
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_blob.py +3 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_bucket.py +65 -4
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_client.py +159 -2
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_transfer_manager.py +1 -6
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/LICENSE +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/MANIFEST.in +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/README.rst +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/__init__.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_http.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/acl.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/batch.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/constants.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/fileio.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/hmac_key.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/iam.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/notification.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/retry.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google_cloud_storage.egg-info/SOURCES.txt +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google_cloud_storage.egg-info/dependency_links.txt +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google_cloud_storage.egg-info/not-zip-safe +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google_cloud_storage.egg-info/requires.txt +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google_cloud_storage.egg-info/top_level.txt +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/setup.cfg +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/setup.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/__init__.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/conformance/README.md +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/conformance/__init__.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/conformance/conftest.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/conformance/retry_strategy_test_data.json +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/conformance/test_conformance.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/data/CloudPlatform_128px_Retina.png +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/data/five-point-one-mb-file.zip +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/data/simple.txt +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/perf/README.md +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/perf/_perf_utils.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/perf/benchmarking.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/perf/profile_w1r3.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/__init__.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_blob.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_bucket.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_fileio.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_hmac_key_metadata.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_kms_integration.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_notification.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/system/test_transfer_manager.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/__init__.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test__helpers.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test__http.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test__signing.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_acl.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_batch.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_fileio.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_hmac_key.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_notification.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/test_retry.py +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/url_signer_v4_test_account.json +0 -0
- {google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/tests/unit/url_signer_v4_test_data.json +0 -0
{google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_helpers.py
RENAMED
|
@@ -293,6 +293,9 @@ class _PropertyMixin(object):
|
|
|
293
293
|
)
|
|
294
294
|
if soft_deleted is not None:
|
|
295
295
|
query_params["softDeleted"] = soft_deleted
|
|
296
|
+
# Soft delete reload requires a generation, even for targets
|
|
297
|
+
# that don't include them in default query params (buckets).
|
|
298
|
+
query_params["generation"] = self.generation
|
|
296
299
|
headers = self._encryption_headers()
|
|
297
300
|
_add_etag_match_headers(
|
|
298
301
|
headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
|
|
@@ -54,6 +54,12 @@ _default_attributes = {
|
|
|
54
54
|
"user_agent.original": f"gcloud-python/{__version__}",
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
_cloud_trace_adoption_attrs = {
|
|
58
|
+
"gcp.client.service": "storage",
|
|
59
|
+
"gcp.client.version": __version__,
|
|
60
|
+
"gcp.client.repo": "googleapis/python-storage",
|
|
61
|
+
}
|
|
62
|
+
|
|
57
63
|
|
|
58
64
|
@contextmanager
|
|
59
65
|
def create_trace_span(name, attributes=None, client=None, api_request=None, retry=None):
|
|
@@ -79,6 +85,7 @@ def create_trace_span(name, attributes=None, client=None, api_request=None, retr
|
|
|
79
85
|
|
|
80
86
|
def _get_final_attributes(attributes=None, client=None, api_request=None, retry=None):
|
|
81
87
|
collected_attr = _default_attributes.copy()
|
|
88
|
+
collected_attr.update(_cloud_trace_adoption_attrs)
|
|
82
89
|
if api_request:
|
|
83
90
|
collected_attr.update(_set_api_request_attr(api_request, client))
|
|
84
91
|
if isinstance(retry, api_retry.Retry):
|
{google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/_signing.py
RENAMED
|
@@ -28,8 +28,10 @@ import google.auth.credentials
|
|
|
28
28
|
from google.auth import exceptions
|
|
29
29
|
from google.auth.transport import requests
|
|
30
30
|
from google.cloud import _helpers
|
|
31
|
+
from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
|
|
31
32
|
from google.cloud.storage._helpers import _NOW
|
|
32
33
|
from google.cloud.storage._helpers import _UTC
|
|
34
|
+
from google.cloud.storage.retry import DEFAULT_RETRY
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
# `google.cloud.storage._signing.NOW` is deprecated.
|
|
@@ -271,6 +273,7 @@ def generate_signed_url_v2(
|
|
|
271
273
|
query_parameters=None,
|
|
272
274
|
service_account_email=None,
|
|
273
275
|
access_token=None,
|
|
276
|
+
universe_domain=None,
|
|
274
277
|
):
|
|
275
278
|
"""Generate a V2 signed URL to provide query-string auth'n to a resource.
|
|
276
279
|
|
|
@@ -384,7 +387,9 @@ def generate_signed_url_v2(
|
|
|
384
387
|
# See https://github.com/googleapis/google-cloud-python/issues/922
|
|
385
388
|
# Set the right query parameters.
|
|
386
389
|
if access_token and service_account_email:
|
|
387
|
-
signature = _sign_message(
|
|
390
|
+
signature = _sign_message(
|
|
391
|
+
string_to_sign, access_token, service_account_email, universe_domain
|
|
392
|
+
)
|
|
388
393
|
signed_query_params = {
|
|
389
394
|
"GoogleAccessId": service_account_email,
|
|
390
395
|
"Expires": expiration_stamp,
|
|
@@ -432,6 +437,7 @@ def generate_signed_url_v4(
|
|
|
432
437
|
query_parameters=None,
|
|
433
438
|
service_account_email=None,
|
|
434
439
|
access_token=None,
|
|
440
|
+
universe_domain=None,
|
|
435
441
|
_request_timestamp=None, # for testing only
|
|
436
442
|
):
|
|
437
443
|
"""Generate a V4 signed URL to provide query-string auth'n to a resource.
|
|
@@ -623,7 +629,9 @@ def generate_signed_url_v4(
|
|
|
623
629
|
string_to_sign = "\n".join(string_elements)
|
|
624
630
|
|
|
625
631
|
if access_token and service_account_email:
|
|
626
|
-
signature = _sign_message(
|
|
632
|
+
signature = _sign_message(
|
|
633
|
+
string_to_sign, access_token, service_account_email, universe_domain
|
|
634
|
+
)
|
|
627
635
|
signature_bytes = base64.b64decode(signature)
|
|
628
636
|
signature = binascii.hexlify(signature_bytes).decode("ascii")
|
|
629
637
|
else:
|
|
@@ -647,7 +655,12 @@ def get_v4_now_dtstamps():
|
|
|
647
655
|
return timestamp, datestamp
|
|
648
656
|
|
|
649
657
|
|
|
650
|
-
def _sign_message(
|
|
658
|
+
def _sign_message(
|
|
659
|
+
message,
|
|
660
|
+
access_token,
|
|
661
|
+
service_account_email,
|
|
662
|
+
universe_domain=_DEFAULT_UNIVERSE_DOMAIN,
|
|
663
|
+
):
|
|
651
664
|
"""Signs a message.
|
|
652
665
|
|
|
653
666
|
:type message: str
|
|
@@ -669,17 +682,22 @@ def _sign_message(message, access_token, service_account_email):
|
|
|
669
682
|
message = _helpers._to_bytes(message)
|
|
670
683
|
|
|
671
684
|
method = "POST"
|
|
672
|
-
url = "https://iamcredentials.
|
|
673
|
-
service_account_email
|
|
674
|
-
)
|
|
685
|
+
url = f"https://iamcredentials.{universe_domain}/v1/projects/-/serviceAccounts/{service_account_email}:signBlob?alt=json"
|
|
675
686
|
headers = {
|
|
676
687
|
"Authorization": "Bearer " + access_token,
|
|
677
688
|
"Content-type": "application/json",
|
|
678
689
|
}
|
|
679
690
|
body = json.dumps({"payload": base64.b64encode(message).decode("utf-8")})
|
|
680
|
-
|
|
681
691
|
request = requests.Request()
|
|
682
|
-
|
|
692
|
+
|
|
693
|
+
def retriable_request():
|
|
694
|
+
response = request(url=url, method=method, body=body, headers=headers)
|
|
695
|
+
return response
|
|
696
|
+
|
|
697
|
+
# Apply the default retry object to the signBlob call.
|
|
698
|
+
retry = DEFAULT_RETRY
|
|
699
|
+
call = retry(retriable_request)
|
|
700
|
+
response = call()
|
|
683
701
|
|
|
684
702
|
if response.status != http.client.OK:
|
|
685
703
|
raise exceptions.TransportError(
|
|
@@ -607,6 +607,9 @@ class Blob(_PropertyMixin):
|
|
|
607
607
|
client = self._require_client(client) # May be redundant, but that's ok.
|
|
608
608
|
credentials = client._credentials
|
|
609
609
|
|
|
610
|
+
client = self._require_client(client)
|
|
611
|
+
universe_domain = client.universe_domain
|
|
612
|
+
|
|
610
613
|
if version == "v2":
|
|
611
614
|
helper = generate_signed_url_v2
|
|
612
615
|
else:
|
|
@@ -638,6 +641,7 @@ class Blob(_PropertyMixin):
|
|
|
638
641
|
query_parameters=query_parameters,
|
|
639
642
|
service_account_email=service_account_email,
|
|
640
643
|
access_token=access_token,
|
|
644
|
+
universe_domain=universe_domain,
|
|
641
645
|
)
|
|
642
646
|
|
|
643
647
|
@create_trace_span(name="Storage.Blob.exists")
|
|
@@ -2344,7 +2348,6 @@ class Blob(_PropertyMixin):
|
|
|
2344
2348
|
"upload.checksum": f"{checksum}",
|
|
2345
2349
|
}
|
|
2346
2350
|
args = {"timeout": timeout}
|
|
2347
|
-
# import pdb; pdb.set_trace()
|
|
2348
2351
|
with create_trace_span(
|
|
2349
2352
|
name="Storage.ResumableUpload/transmitNextChunk",
|
|
2350
2353
|
attributes=extra_attributes,
|
|
@@ -626,6 +626,10 @@ class Bucket(_PropertyMixin):
|
|
|
626
626
|
:type user_project: str
|
|
627
627
|
:param user_project: (Optional) the project ID to be billed for API
|
|
628
628
|
requests made via this instance.
|
|
629
|
+
|
|
630
|
+
:type generation: int
|
|
631
|
+
:param generation: (Optional) If present, selects a specific revision of
|
|
632
|
+
this bucket.
|
|
629
633
|
"""
|
|
630
634
|
|
|
631
635
|
_MAX_OBJECTS_FOR_ITERATION = 256
|
|
@@ -659,7 +663,7 @@ class Bucket(_PropertyMixin):
|
|
|
659
663
|
)
|
|
660
664
|
"""Allowed values for :attr:`location_type`."""
|
|
661
665
|
|
|
662
|
-
def __init__(self, client, name=None, user_project=None):
|
|
666
|
+
def __init__(self, client, name=None, user_project=None, generation=None):
|
|
663
667
|
"""
|
|
664
668
|
property :attr:`name`
|
|
665
669
|
Get the bucket's name.
|
|
@@ -672,6 +676,9 @@ class Bucket(_PropertyMixin):
|
|
|
672
676
|
self._label_removals = set()
|
|
673
677
|
self._user_project = user_project
|
|
674
678
|
|
|
679
|
+
if generation is not None:
|
|
680
|
+
self._properties["generation"] = generation
|
|
681
|
+
|
|
675
682
|
def __repr__(self):
|
|
676
683
|
return f"<Bucket: {self.name}>"
|
|
677
684
|
|
|
@@ -726,6 +733,50 @@ class Bucket(_PropertyMixin):
|
|
|
726
733
|
"""
|
|
727
734
|
return self._user_project
|
|
728
735
|
|
|
736
|
+
@property
|
|
737
|
+
def generation(self):
|
|
738
|
+
"""Retrieve the generation for the bucket.
|
|
739
|
+
|
|
740
|
+
:rtype: int or ``NoneType``
|
|
741
|
+
:returns: The generation of the bucket or ``None`` if the bucket's
|
|
742
|
+
resource has not been loaded from the server.
|
|
743
|
+
"""
|
|
744
|
+
generation = self._properties.get("generation")
|
|
745
|
+
if generation is not None:
|
|
746
|
+
return int(generation)
|
|
747
|
+
|
|
748
|
+
@property
|
|
749
|
+
def soft_delete_time(self):
|
|
750
|
+
"""If this bucket has been soft-deleted, returns the time at which it became soft-deleted.
|
|
751
|
+
|
|
752
|
+
:rtype: :class:`datetime.datetime` or ``NoneType``
|
|
753
|
+
:returns:
|
|
754
|
+
(readonly) The time that the bucket became soft-deleted.
|
|
755
|
+
Note this property is only set for soft-deleted buckets.
|
|
756
|
+
"""
|
|
757
|
+
soft_delete_time = self._properties.get("softDeleteTime")
|
|
758
|
+
if soft_delete_time is not None:
|
|
759
|
+
return _rfc3339_nanos_to_datetime(soft_delete_time)
|
|
760
|
+
|
|
761
|
+
@property
|
|
762
|
+
def hard_delete_time(self):
|
|
763
|
+
"""If this bucket has been soft-deleted, returns the time at which it will be permanently deleted.
|
|
764
|
+
|
|
765
|
+
:rtype: :class:`datetime.datetime` or ``NoneType``
|
|
766
|
+
:returns:
|
|
767
|
+
(readonly) The time that the bucket will be permanently deleted.
|
|
768
|
+
Note this property is only set for soft-deleted buckets.
|
|
769
|
+
"""
|
|
770
|
+
hard_delete_time = self._properties.get("hardDeleteTime")
|
|
771
|
+
if hard_delete_time is not None:
|
|
772
|
+
return _rfc3339_nanos_to_datetime(hard_delete_time)
|
|
773
|
+
|
|
774
|
+
@property
|
|
775
|
+
def _query_params(self):
|
|
776
|
+
"""Default query parameters."""
|
|
777
|
+
params = super()._query_params
|
|
778
|
+
return params
|
|
779
|
+
|
|
729
780
|
@classmethod
|
|
730
781
|
def from_string(cls, uri, client=None):
|
|
731
782
|
"""Get a constructor for bucket object by URI.
|
|
@@ -1045,6 +1096,7 @@ class Bucket(_PropertyMixin):
|
|
|
1045
1096
|
if_metageneration_match=None,
|
|
1046
1097
|
if_metageneration_not_match=None,
|
|
1047
1098
|
retry=DEFAULT_RETRY,
|
|
1099
|
+
soft_deleted=None,
|
|
1048
1100
|
):
|
|
1049
1101
|
"""Reload properties from Cloud Storage.
|
|
1050
1102
|
|
|
@@ -1084,6 +1136,13 @@ class Bucket(_PropertyMixin):
|
|
|
1084
1136
|
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
|
|
1085
1137
|
:param retry:
|
|
1086
1138
|
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
|
|
1139
|
+
|
|
1140
|
+
:type soft_deleted: bool
|
|
1141
|
+
:param soft_deleted: (Optional) If True, looks for a soft-deleted
|
|
1142
|
+
bucket. Will only return the bucket metadata if the bucket exists
|
|
1143
|
+
and is in a soft-deleted state. The bucket ``generation`` must be
|
|
1144
|
+
set if ``soft_deleted`` is set to True.
|
|
1145
|
+
See: https://cloud.google.com/storage/docs/soft-delete
|
|
1087
1146
|
"""
|
|
1088
1147
|
super(Bucket, self).reload(
|
|
1089
1148
|
client=client,
|
|
@@ -1094,6 +1153,7 @@ class Bucket(_PropertyMixin):
|
|
|
1094
1153
|
if_metageneration_match=if_metageneration_match,
|
|
1095
1154
|
if_metageneration_not_match=if_metageneration_not_match,
|
|
1096
1155
|
retry=retry,
|
|
1156
|
+
soft_deleted=soft_deleted,
|
|
1097
1157
|
)
|
|
1098
1158
|
|
|
1099
1159
|
@create_trace_span(name="Storage.Bucket.patch")
|
|
@@ -2159,8 +2219,8 @@ class Bucket(_PropertyMixin):
|
|
|
2159
2219
|
:param client: (Optional) The client to use. If not passed, falls back
|
|
2160
2220
|
to the ``client`` stored on the current bucket.
|
|
2161
2221
|
|
|
2162
|
-
:type generation:
|
|
2163
|
-
:param generation:
|
|
2222
|
+
:type generation: int
|
|
2223
|
+
:param generation: Selects the specific revision of the object.
|
|
2164
2224
|
|
|
2165
2225
|
:type copy_source_acl: bool
|
|
2166
2226
|
:param copy_source_acl: (Optional) If true, copy the soft-deleted object's access controls.
|
|
@@ -30,6 +30,7 @@ from google.cloud._helpers import _LocalStack
|
|
|
30
30
|
from google.cloud.client import ClientWithProject
|
|
31
31
|
from google.cloud.exceptions import NotFound
|
|
32
32
|
|
|
33
|
+
from google.cloud.storage._helpers import _add_generation_match_parameters
|
|
33
34
|
from google.cloud.storage._helpers import _bucket_bound_hostname_url
|
|
34
35
|
from google.cloud.storage._helpers import _get_api_endpoint_override
|
|
35
36
|
from google.cloud.storage._helpers import _get_environ_project
|
|
@@ -367,7 +368,7 @@ class Client(ClientWithProject):
|
|
|
367
368
|
api_response = self._get_resource(path, timeout=timeout, retry=retry)
|
|
368
369
|
return api_response["email_address"]
|
|
369
370
|
|
|
370
|
-
def bucket(self, bucket_name, user_project=None):
|
|
371
|
+
def bucket(self, bucket_name, user_project=None, generation=None):
|
|
371
372
|
"""Factory constructor for bucket object.
|
|
372
373
|
|
|
373
374
|
.. note::
|
|
@@ -381,10 +382,19 @@ class Client(ClientWithProject):
|
|
|
381
382
|
:param user_project: (Optional) The project ID to be billed for API
|
|
382
383
|
requests made via the bucket.
|
|
383
384
|
|
|
385
|
+
:type generation: int
|
|
386
|
+
:param generation: (Optional) If present, selects a specific revision of
|
|
387
|
+
this bucket.
|
|
388
|
+
|
|
384
389
|
:rtype: :class:`google.cloud.storage.bucket.Bucket`
|
|
385
390
|
:returns: The bucket object created.
|
|
386
391
|
"""
|
|
387
|
-
return Bucket(
|
|
392
|
+
return Bucket(
|
|
393
|
+
client=self,
|
|
394
|
+
name=bucket_name,
|
|
395
|
+
user_project=user_project,
|
|
396
|
+
generation=generation,
|
|
397
|
+
)
|
|
388
398
|
|
|
389
399
|
def batch(self, raise_exception=True):
|
|
390
400
|
"""Factory constructor for batch object.
|
|
@@ -789,7 +799,7 @@ class Client(ClientWithProject):
|
|
|
789
799
|
_target_object=_target_object,
|
|
790
800
|
)
|
|
791
801
|
|
|
792
|
-
def _bucket_arg_to_bucket(self, bucket_or_name):
|
|
802
|
+
def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
|
|
793
803
|
"""Helper to return given bucket or create new by name.
|
|
794
804
|
|
|
795
805
|
Args:
|
|
@@ -798,17 +808,27 @@ class Client(ClientWithProject):
|
|
|
798
808
|
str, \
|
|
799
809
|
]):
|
|
800
810
|
The bucket resource to pass or name to create.
|
|
811
|
+
generation (Optional[int]):
|
|
812
|
+
The bucket generation. If generation is specified,
|
|
813
|
+
bucket_or_name must be a name (str).
|
|
801
814
|
|
|
802
815
|
Returns:
|
|
803
816
|
google.cloud.storage.bucket.Bucket
|
|
804
817
|
The newly created bucket or the given one.
|
|
805
818
|
"""
|
|
806
819
|
if isinstance(bucket_or_name, Bucket):
|
|
820
|
+
if generation:
|
|
821
|
+
raise ValueError(
|
|
822
|
+
"The generation can only be specified if a "
|
|
823
|
+
"name is used to specify a bucket, not a Bucket object. "
|
|
824
|
+
"Create a new Bucket object with the correct generation "
|
|
825
|
+
"instead."
|
|
826
|
+
)
|
|
807
827
|
bucket = bucket_or_name
|
|
808
828
|
if bucket.client is None:
|
|
809
829
|
bucket._client = self
|
|
810
830
|
else:
|
|
811
|
-
bucket = Bucket(self, name=bucket_or_name)
|
|
831
|
+
bucket = Bucket(self, name=bucket_or_name, generation=generation)
|
|
812
832
|
return bucket
|
|
813
833
|
|
|
814
834
|
@create_trace_span(name="Storage.Client.getBucket")
|
|
@@ -819,6 +839,9 @@ class Client(ClientWithProject):
|
|
|
819
839
|
if_metageneration_match=None,
|
|
820
840
|
if_metageneration_not_match=None,
|
|
821
841
|
retry=DEFAULT_RETRY,
|
|
842
|
+
*,
|
|
843
|
+
generation=None,
|
|
844
|
+
soft_deleted=None,
|
|
822
845
|
):
|
|
823
846
|
"""Retrieve a bucket via a GET request.
|
|
824
847
|
|
|
@@ -837,12 +860,12 @@ class Client(ClientWithProject):
|
|
|
837
860
|
Can also be passed as a tuple (connect_timeout, read_timeout).
|
|
838
861
|
See :meth:`requests.Session.request` documentation for details.
|
|
839
862
|
|
|
840
|
-
if_metageneration_match (Optional[
|
|
863
|
+
if_metageneration_match (Optional[int]):
|
|
841
864
|
Make the operation conditional on whether the
|
|
842
|
-
|
|
865
|
+
bucket's current metageneration matches the given value.
|
|
843
866
|
|
|
844
|
-
if_metageneration_not_match (Optional[
|
|
845
|
-
Make the operation conditional on whether the
|
|
867
|
+
if_metageneration_not_match (Optional[int]):
|
|
868
|
+
Make the operation conditional on whether the bucket's
|
|
846
869
|
current metageneration does not match the given value.
|
|
847
870
|
|
|
848
871
|
retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
|
|
@@ -859,6 +882,19 @@ class Client(ClientWithProject):
|
|
|
859
882
|
See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
|
|
860
883
|
information on retry types and how to configure them.
|
|
861
884
|
|
|
885
|
+
generation (Optional[int]):
|
|
886
|
+
The generation of the bucket. The generation can be used to
|
|
887
|
+
specify a specific soft-deleted version of the bucket, in
|
|
888
|
+
conjunction with the ``soft_deleted`` argument below. If
|
|
889
|
+
``soft_deleted`` is not True, the generation is unused.
|
|
890
|
+
|
|
891
|
+
soft_deleted (Optional[bool]):
|
|
892
|
+
If True, looks for a soft-deleted bucket. Will only return
|
|
893
|
+
the bucket metadata if the bucket exists and is in a
|
|
894
|
+
soft-deleted state. The bucket ``generation`` is required if
|
|
895
|
+
``soft_deleted`` is set to True.
|
|
896
|
+
See: https://cloud.google.com/storage/docs/soft-delete
|
|
897
|
+
|
|
862
898
|
Returns:
|
|
863
899
|
google.cloud.storage.bucket.Bucket
|
|
864
900
|
The bucket matching the name provided.
|
|
@@ -867,13 +903,14 @@ class Client(ClientWithProject):
|
|
|
867
903
|
google.cloud.exceptions.NotFound
|
|
868
904
|
If the bucket is not found.
|
|
869
905
|
"""
|
|
870
|
-
bucket = self._bucket_arg_to_bucket(bucket_or_name)
|
|
906
|
+
bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
|
|
871
907
|
bucket.reload(
|
|
872
908
|
client=self,
|
|
873
909
|
timeout=timeout,
|
|
874
910
|
if_metageneration_match=if_metageneration_match,
|
|
875
911
|
if_metageneration_not_match=if_metageneration_not_match,
|
|
876
912
|
retry=retry,
|
|
913
|
+
soft_deleted=soft_deleted,
|
|
877
914
|
)
|
|
878
915
|
return bucket
|
|
879
916
|
|
|
@@ -1386,6 +1423,8 @@ class Client(ClientWithProject):
|
|
|
1386
1423
|
page_size=None,
|
|
1387
1424
|
timeout=_DEFAULT_TIMEOUT,
|
|
1388
1425
|
retry=DEFAULT_RETRY,
|
|
1426
|
+
*,
|
|
1427
|
+
soft_deleted=None,
|
|
1389
1428
|
):
|
|
1390
1429
|
"""Get all buckets in the project associated to the client.
|
|
1391
1430
|
|
|
@@ -1438,6 +1477,12 @@ class Client(ClientWithProject):
|
|
|
1438
1477
|
:param retry:
|
|
1439
1478
|
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
|
|
1440
1479
|
|
|
1480
|
+
:type soft_deleted: bool
|
|
1481
|
+
:param soft_deleted:
|
|
1482
|
+
(Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
|
|
1483
|
+
generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
|
|
1484
|
+
See: https://cloud.google.com/storage/docs/soft-delete
|
|
1485
|
+
|
|
1441
1486
|
:rtype: :class:`~google.api_core.page_iterator.Iterator`
|
|
1442
1487
|
:raises ValueError: if both ``project`` is ``None`` and the client's
|
|
1443
1488
|
project is also ``None``.
|
|
@@ -1469,6 +1514,9 @@ class Client(ClientWithProject):
|
|
|
1469
1514
|
if fields is not None:
|
|
1470
1515
|
extra_params["fields"] = fields
|
|
1471
1516
|
|
|
1517
|
+
if soft_deleted is not None:
|
|
1518
|
+
extra_params["softDeleted"] = soft_deleted
|
|
1519
|
+
|
|
1472
1520
|
return self._list_resource(
|
|
1473
1521
|
"/b",
|
|
1474
1522
|
_item_to_bucket,
|
|
@@ -1480,6 +1528,71 @@ class Client(ClientWithProject):
|
|
|
1480
1528
|
retry=retry,
|
|
1481
1529
|
)
|
|
1482
1530
|
|
|
1531
|
+
def restore_bucket(
|
|
1532
|
+
self,
|
|
1533
|
+
bucket_name,
|
|
1534
|
+
generation,
|
|
1535
|
+
projection="noAcl",
|
|
1536
|
+
if_metageneration_match=None,
|
|
1537
|
+
if_metageneration_not_match=None,
|
|
1538
|
+
timeout=_DEFAULT_TIMEOUT,
|
|
1539
|
+
retry=DEFAULT_RETRY,
|
|
1540
|
+
):
|
|
1541
|
+
"""Restores a soft-deleted bucket.
|
|
1542
|
+
|
|
1543
|
+
:type bucket_name: str
|
|
1544
|
+
:param bucket_name: The name of the bucket to be restored.
|
|
1545
|
+
|
|
1546
|
+
:type generation: int
|
|
1547
|
+
:param generation: Selects the specific revision of the bucket.
|
|
1548
|
+
|
|
1549
|
+
:type projection: str
|
|
1550
|
+
:param projection:
|
|
1551
|
+
(Optional) Specifies the set of properties to return. If used, must
|
|
1552
|
+
be 'full' or 'noAcl'. Defaults to 'noAcl'.
|
|
1553
|
+
|
|
1554
|
+
if_metageneration_match (Optional[int]):
|
|
1555
|
+
Make the operation conditional on whether the
|
|
1556
|
+
blob's current metageneration matches the given value.
|
|
1557
|
+
|
|
1558
|
+
if_metageneration_not_match (Optional[int]):
|
|
1559
|
+
Make the operation conditional on whether the blob's
|
|
1560
|
+
current metageneration does not match the given value.
|
|
1561
|
+
|
|
1562
|
+
:type timeout: float or tuple
|
|
1563
|
+
:param timeout:
|
|
1564
|
+
(Optional) The amount of time, in seconds, to wait
|
|
1565
|
+
for the server response. See: :ref:`configuring_timeouts`
|
|
1566
|
+
|
|
1567
|
+
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
|
|
1568
|
+
:param retry:
|
|
1569
|
+
(Optional) How to retry the RPC.
|
|
1570
|
+
|
|
1571
|
+
Users can configure non-default retry behavior. A ``None`` value will
|
|
1572
|
+
disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
|
|
1573
|
+
|
|
1574
|
+
:rtype: :class:`google.cloud.storage.bucket.Bucket`
|
|
1575
|
+
:returns: The restored Bucket.
|
|
1576
|
+
"""
|
|
1577
|
+
query_params = {"generation": generation, "projection": projection}
|
|
1578
|
+
|
|
1579
|
+
_add_generation_match_parameters(
|
|
1580
|
+
query_params,
|
|
1581
|
+
if_metageneration_match=if_metageneration_match,
|
|
1582
|
+
if_metageneration_not_match=if_metageneration_not_match,
|
|
1583
|
+
)
|
|
1584
|
+
|
|
1585
|
+
bucket = self.bucket(bucket_name)
|
|
1586
|
+
api_response = self._post_resource(
|
|
1587
|
+
f"{bucket.path}/restore",
|
|
1588
|
+
None,
|
|
1589
|
+
query_params=query_params,
|
|
1590
|
+
timeout=timeout,
|
|
1591
|
+
retry=retry,
|
|
1592
|
+
)
|
|
1593
|
+
bucket._set_properties(api_response)
|
|
1594
|
+
return bucket
|
|
1595
|
+
|
|
1483
1596
|
@create_trace_span(name="Storage.Client.createHmacKey")
|
|
1484
1597
|
def create_hmac_key(
|
|
1485
1598
|
self,
|
|
@@ -1724,13 +1837,16 @@ class Client(ClientWithProject):
|
|
|
1724
1837
|
)
|
|
1725
1838
|
|
|
1726
1839
|
credentials = self._credentials if credentials is None else credentials
|
|
1727
|
-
|
|
1840
|
+
client_email = service_account_email
|
|
1841
|
+
if not access_token or not service_account_email:
|
|
1842
|
+
ensure_signed_credentials(credentials)
|
|
1843
|
+
client_email = credentials.signer_email
|
|
1728
1844
|
|
|
1729
1845
|
# prepare policy conditions and fields
|
|
1730
1846
|
timestamp, datestamp = get_v4_now_dtstamps()
|
|
1731
1847
|
|
|
1732
1848
|
x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
|
|
1733
|
-
email=
|
|
1849
|
+
email=client_email, datestamp=datestamp
|
|
1734
1850
|
)
|
|
1735
1851
|
required_conditions = [
|
|
1736
1852
|
{"bucket": bucket_name},
|
{google_cloud_storage-2.18.2 → google_cloud_storage-2.19.0}/google/cloud/storage/transfer_manager.py
RENAMED
|
@@ -885,6 +885,8 @@ def download_chunks_concurrently(
|
|
|
885
885
|
"'checksum' is in download_kwargs, but is not supported because sliced downloads have a different checksum mechanism from regular downloads. Use the 'crc32c_checksum' argument on download_chunks_concurrently instead."
|
|
886
886
|
)
|
|
887
887
|
|
|
888
|
+
download_kwargs = download_kwargs.copy()
|
|
889
|
+
download_kwargs["checksum"] = None
|
|
888
890
|
download_kwargs["command"] = "tm.download_sharded"
|
|
889
891
|
|
|
890
892
|
# We must know the size and the generation of the blob.
|
|
@@ -31,6 +31,10 @@ retry_failures = RetryErrors(AssertionError)
|
|
|
31
31
|
|
|
32
32
|
user_project = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT")
|
|
33
33
|
testing_mtls = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
|
|
34
|
+
test_universe_domain = os.getenv("TEST_UNIVERSE_DOMAIN")
|
|
35
|
+
test_universe_project_id = os.getenv("TEST_UNIVERSE_PROJECT_ID")
|
|
36
|
+
test_universe_location = os.getenv("TEST_UNIVERSE_LOCATION")
|
|
37
|
+
test_universe_domain_credential = os.getenv("TEST_UNIVERSE_DOMAIN_CREDENTIAL")
|
|
34
38
|
signing_blob_content = b"This time for sure, Rocky!"
|
|
35
39
|
is_api_endpoint_override = (
|
|
36
40
|
_get_default_storage_base_url() != "https://storage.googleapis.com"
|
|
@@ -331,3 +331,86 @@ def keyring(storage_client, kms_bucket, kms_client):
|
|
|
331
331
|
except exceptions.NotFound:
|
|
332
332
|
key = {"purpose": purpose}
|
|
333
333
|
kms_client.create_crypto_key(keyring_path, key_name, key)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
@pytest.fixture(scope="function")
|
|
337
|
+
def test_universe_domain():
|
|
338
|
+
if _helpers.test_universe_domain is None:
|
|
339
|
+
pytest.skip("TEST_UNIVERSE_DOMAIN not set in environment.")
|
|
340
|
+
return _helpers.test_universe_domain
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@pytest.fixture(scope="function")
|
|
344
|
+
def test_universe_project_id():
|
|
345
|
+
if _helpers.test_universe_project_id is None:
|
|
346
|
+
pytest.skip("TEST_UNIVERSE_PROJECT_ID not set in environment.")
|
|
347
|
+
return _helpers.test_universe_project_id
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
@pytest.fixture(scope="function")
|
|
351
|
+
def test_universe_location():
|
|
352
|
+
if _helpers.test_universe_location is None:
|
|
353
|
+
pytest.skip("TEST_UNIVERSE_LOCATION not set in environment.")
|
|
354
|
+
return _helpers.test_universe_location
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
@pytest.fixture(scope="function")
|
|
358
|
+
def test_universe_domain_credential():
|
|
359
|
+
if _helpers.test_universe_domain_credential is None:
|
|
360
|
+
pytest.skip("TEST_UNIVERSE_DOMAIN_CREDENTIAL not set in environment.")
|
|
361
|
+
return _helpers.test_universe_domain_credential
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@pytest.fixture(scope="function")
|
|
365
|
+
def universe_domain_credential(test_universe_domain_credential):
|
|
366
|
+
from google.oauth2 import service_account
|
|
367
|
+
|
|
368
|
+
return service_account.Credentials.from_service_account_file(
|
|
369
|
+
test_universe_domain_credential
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@pytest.fixture(scope="function")
|
|
374
|
+
def universe_domain_client(
|
|
375
|
+
test_universe_domain, test_universe_project_id, universe_domain_credential
|
|
376
|
+
):
|
|
377
|
+
from google.cloud.storage import Client
|
|
378
|
+
|
|
379
|
+
client_options = {"universe_domain": test_universe_domain}
|
|
380
|
+
ud_storage_client = Client(
|
|
381
|
+
project=test_universe_project_id,
|
|
382
|
+
credentials=universe_domain_credential,
|
|
383
|
+
client_options=client_options,
|
|
384
|
+
)
|
|
385
|
+
with contextlib.closing(ud_storage_client):
|
|
386
|
+
yield ud_storage_client
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
@pytest.fixture(scope="function")
|
|
390
|
+
def universe_domain_bucket(universe_domain_client, test_universe_location):
|
|
391
|
+
bucket_name = _helpers.unique_name("gcp-systest-ud")
|
|
392
|
+
bucket = universe_domain_client.create_bucket(
|
|
393
|
+
bucket_name, location=test_universe_location
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
blob = bucket.blob("README.txt")
|
|
397
|
+
blob.upload_from_string(_helpers.signing_blob_content)
|
|
398
|
+
|
|
399
|
+
yield bucket
|
|
400
|
+
|
|
401
|
+
_helpers.delete_bucket(bucket)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@pytest.fixture(scope="function")
|
|
405
|
+
def universe_domain_iam_client(
|
|
406
|
+
test_universe_domain, test_universe_project_id, universe_domain_credential
|
|
407
|
+
):
|
|
408
|
+
from google.cloud import iam_credentials_v1
|
|
409
|
+
|
|
410
|
+
client_options = {"universe_domain": test_universe_domain}
|
|
411
|
+
iam_client = iam_credentials_v1.IAMCredentialsClient(
|
|
412
|
+
credentials=universe_domain_credential,
|
|
413
|
+
client_options=client_options,
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
return iam_client
|