google-cloud-storage 3.1.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {google_cloud_storage-3.1.0/google_cloud_storage.egg-info → google_cloud_storage-3.2.0}/PKG-INFO +117 -80
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/README.rst +94 -70
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/_download.py +6 -1
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/requests/download.py +85 -26
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/blob.py +61 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/client.py +5 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/fileio.py +7 -3
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/notification.py +1 -1
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/version.py +1 -1
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0/google_cloud_storage.egg-info}/PKG-INFO +117 -80
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google_cloud_storage.egg-info/SOURCES.txt +0 -1
- google_cloud_storage-3.2.0/google_cloud_storage.egg-info/requires.txt +12 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/setup.cfg +0 -3
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/setup.py +9 -8
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/requests/test_download.py +43 -1
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/requests/test_download.py +109 -4
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_blob.py +19 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_notification.py +3 -3
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_blob.py +74 -7
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_bucket.py +1 -1
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_client.py +3 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_fileio.py +4 -2
- google_cloud_storage-3.1.0/google_cloud_storage.egg-info/requires.txt +0 -12
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/LICENSE +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/MANIFEST.in +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_http.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/_helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/_upload.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/common.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/py.typed +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/requests/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/requests/_request_helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/requests/upload.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_opentelemetry_tracing.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_signing.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/acl.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/batch.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/bucket.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/constants.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/exceptions.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/hmac_key.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/iam.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/retry.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/transfer_manager.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google_cloud_storage.egg-info/dependency_links.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google_cloud_storage.egg-info/not-zip-safe +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google_cloud_storage.egg-info/top_level.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/conformance/README.md +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/conformance/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/conformance/conftest.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/conformance/retry_strategy_test_data.json +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/conformance/test_conformance.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/data/CloudPlatform_128px_Retina.png +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/data/five-point-one-mb-file.zip +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/data/simple.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/perf/README.md +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/perf/_perf_utils.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/perf/benchmarking.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/perf/profile_w1r3.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/brotli.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/brotli.txt.br +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/favicon.ico +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/file.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/gzipped.txt +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/gzipped.txt.gz +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/image1.jpg +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/data/image2.jpg +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/credentials.json.enc +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/requests/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/requests/conftest.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/requests/test_upload.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/system/utils.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/requests/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/requests/test__helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/requests/test_upload.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/test__download.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/test__helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/resumable_media/unit/test__upload.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/_helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/conftest.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test__signing.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_bucket.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_client.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_fileio.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_hmac_key_metadata.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_kms_integration.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/system/test_transfer_manager.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/__init__.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test__helpers.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test__http.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test__opentelemetry_tracing.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test__signing.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_acl.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_batch.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_exceptions.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_hmac_key.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_notification.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_retry.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/test_transfer_manager.py +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/url_signer_v4_test_account.json +0 -0
- {google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/tests/unit/url_signer_v4_test_data.json +0 -0
{google_cloud_storage-3.1.0/google_cloud_storage.egg-info → google_cloud_storage-3.2.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: google-cloud-storage
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Google Cloud Storage API client library
|
|
5
5
|
Home-page: https://github.com/googleapis/python-storage
|
|
6
6
|
Author: Google LLC
|
|
@@ -18,20 +18,33 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
22
|
Classifier: Operating System :: OS Independent
|
|
22
23
|
Classifier: Topic :: Internet
|
|
23
24
|
Requires-Python: >=3.7
|
|
24
25
|
License-File: LICENSE
|
|
25
|
-
Requires-Dist: google-auth<3.
|
|
26
|
-
Requires-Dist: google-api-core<3.0.
|
|
27
|
-
Requires-Dist: google-cloud-core<3.
|
|
28
|
-
Requires-Dist: google-resumable-media
|
|
29
|
-
Requires-Dist: requests<3.0.
|
|
30
|
-
Requires-Dist: google-crc32c<2.
|
|
26
|
+
Requires-Dist: google-auth<3.0.0,>=2.26.1
|
|
27
|
+
Requires-Dist: google-api-core<3.0.0,>=2.15.0
|
|
28
|
+
Requires-Dist: google-cloud-core<3.0.0,>=2.4.2
|
|
29
|
+
Requires-Dist: google-resumable-media<3.0.0,>=2.7.2
|
|
30
|
+
Requires-Dist: requests<3.0.0,>=2.22.0
|
|
31
|
+
Requires-Dist: google-crc32c<2.0.0,>=1.1.3
|
|
31
32
|
Provides-Extra: protobuf
|
|
32
|
-
Requires-Dist: protobuf<
|
|
33
|
+
Requires-Dist: protobuf<7.0.0,>=3.20.2; extra == "protobuf"
|
|
33
34
|
Provides-Extra: tracing
|
|
34
|
-
Requires-Dist: opentelemetry-api
|
|
35
|
+
Requires-Dist: opentelemetry-api<2.0.0,>=1.1.0; extra == "tracing"
|
|
36
|
+
Dynamic: author
|
|
37
|
+
Dynamic: author-email
|
|
38
|
+
Dynamic: classifier
|
|
39
|
+
Dynamic: description
|
|
40
|
+
Dynamic: home-page
|
|
41
|
+
Dynamic: license
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
Dynamic: platform
|
|
44
|
+
Dynamic: provides-extra
|
|
45
|
+
Dynamic: requires-dist
|
|
46
|
+
Dynamic: requires-python
|
|
47
|
+
Dynamic: summary
|
|
35
48
|
|
|
36
49
|
Python Client for Google Cloud Storage
|
|
37
50
|
======================================
|
|
@@ -43,6 +56,8 @@ allows world-wide storage and retrieval of any amount of data at any time. You c
|
|
|
43
56
|
Cloud Storage for a range of scenarios including serving website content, storing data
|
|
44
57
|
for archival and disaster recovery, or distributing large data objects to users via direct download.
|
|
45
58
|
|
|
59
|
+
**NOTE**: `3.0 Major Version Notes`_ are available. Feedback welcome.
|
|
60
|
+
|
|
46
61
|
A comprehensive list of changes in each version may be found in the `CHANGELOG`_.
|
|
47
62
|
|
|
48
63
|
- `Product Documentation`_
|
|
@@ -64,7 +79,7 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
|
|
|
64
79
|
.. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-storage.svg
|
|
65
80
|
:target: https://pypi.org/project/google-cloud-storage/
|
|
66
81
|
.. _Google Cloud Storage: https://cloud.google.com/storage
|
|
67
|
-
.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/storage/latest
|
|
82
|
+
.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/storage/latest/summary_overview
|
|
68
83
|
.. _Product Documentation: https://cloud.google.com/storage
|
|
69
84
|
.. _CHANGELOG: https://github.com/googleapis/python-storage/blob/main/CHANGELOG.md
|
|
70
85
|
.. _github.com/googleapis/python-storage: https://github.com/googleapis/python-storage
|
|
@@ -72,75 +87,6 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
|
|
|
72
87
|
.. _Storage Control API: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2
|
|
73
88
|
.. _Client Libraries Explained: https://cloud.google.com/apis/docs/client-libraries-explained
|
|
74
89
|
|
|
75
|
-
3.0 Major Version Notes
|
|
76
|
-
-----------------------
|
|
77
|
-
|
|
78
|
-
Feedback Welcome
|
|
79
|
-
~~~~~~~~~~~~~~~~
|
|
80
|
-
|
|
81
|
-
If you experience that backwards compatibility for your application is broken
|
|
82
|
-
with this major version release, please let us know through the Github issues
|
|
83
|
-
system. While some breaks of backwards compatibility may be unavoidable due to
|
|
84
|
-
new features in the major version release, we will do our best to minimize
|
|
85
|
-
them. Thank you.
|
|
86
|
-
|
|
87
|
-
Exception Handling
|
|
88
|
-
~~~~~~~~~~~~~~~~~~
|
|
89
|
-
|
|
90
|
-
In Python Storage 3.0, the dependency ``google-resumable-media`` was integrated.
|
|
91
|
-
The ``google-resumable-media`` dependency included exceptions
|
|
92
|
-
``google.resumable_media.common.InvalidResponse`` and
|
|
93
|
-
``google.resumable_media.common.DataCorruption``, which were often imported
|
|
94
|
-
directly in user application code. The replacements for these exceptions are
|
|
95
|
-
``google.cloud.storage.exceptions.InvalidResponse`` and
|
|
96
|
-
``google.cloud.storage.exceptions.DataCorruption``. Please update application code
|
|
97
|
-
to import and use these exceptions instead.
|
|
98
|
-
|
|
99
|
-
For backwards compatibility, if ``google-resumable-media`` is installed, the new
|
|
100
|
-
exceptions will be defined as subclasses of the old exceptions, so applications
|
|
101
|
-
should continue to work without modification. This backwards compatibility
|
|
102
|
-
feature may be removed in a future major version update.
|
|
103
|
-
|
|
104
|
-
Some users may be using the original exception classes from the
|
|
105
|
-
``google-resumable-media`` library without explicitly installing that library. So
|
|
106
|
-
as not to break user applications following this pattern,
|
|
107
|
-
``google-resumable-media`` is still in the list of dependencies in this package's
|
|
108
|
-
setup.py file. Applications which do not import directly from
|
|
109
|
-
``google-resumable-media`` can safely disregard this dependency.
|
|
110
|
-
This backwards compatibility feature **will be removed** in a future major
|
|
111
|
-
version update. Please migrate to using the ``google.cloud.storage.exceptions``
|
|
112
|
-
classes as above.
|
|
113
|
-
|
|
114
|
-
Checksum Defaults
|
|
115
|
-
~~~~~~~~~~~~~~~~~
|
|
116
|
-
|
|
117
|
-
In Python Storage 3.0, uploads and downloads now have a default of "auto" where
|
|
118
|
-
applicable. "Auto" will use crc32c checksums, except for unusual cases where the
|
|
119
|
-
fast (C extension) crc32c implementation is not available, in which case it will
|
|
120
|
-
use md5 instead. Before Python Storage 3.0, the default was md5 for most
|
|
121
|
-
downloads and None for most uploads. Note that ranged downloads ("start" or
|
|
122
|
-
"end" set) still do not support any checksumming, and some features in
|
|
123
|
-
``transfer_manager.py`` still support crc32c only.
|
|
124
|
-
|
|
125
|
-
Note: The method ``Blob.upload_from_file()`` requires a file in bytes mode, but
|
|
126
|
-
when checksum is set to None, as was the previous default, would not throw an
|
|
127
|
-
error if passed a file in string mode under some circumstances. With the new
|
|
128
|
-
defaults, it will now raise a TypeError. Please use a file opened in bytes
|
|
129
|
-
reading mode as required.
|
|
130
|
-
|
|
131
|
-
Miscellaneous
|
|
132
|
-
~~~~~~~~~~~~~
|
|
133
|
-
|
|
134
|
-
- The ``BlobWriter`` class now attempts to terminate an ongoing resumable upload if
|
|
135
|
-
the writer exits with an exception.
|
|
136
|
-
- Retry behavior is now identical between media operations (uploads and
|
|
137
|
-
downloads) and other operations, and custom predicates are now supported for
|
|
138
|
-
media operations as well.
|
|
139
|
-
- ``Blob.download_as_filename()`` will now delete the empty file if it results in a
|
|
140
|
-
google.cloud.exceptions.NotFound exception (HTTP 404).
|
|
141
|
-
- Previously, object upload, metadata update, and delete methods had retries
|
|
142
|
-
disabled by default unless the generation or metageneration was specified in
|
|
143
|
-
the request. This has now changed so that retries are enabled by default.
|
|
144
90
|
|
|
145
91
|
Quick Start
|
|
146
92
|
-----------
|
|
@@ -221,6 +167,26 @@ Windows
|
|
|
221
167
|
pip install google-cloud-storage
|
|
222
168
|
|
|
223
169
|
|
|
170
|
+
Example Usage
|
|
171
|
+
~~~~~~~~~~~~~
|
|
172
|
+
|
|
173
|
+
.. code-block:: python
|
|
174
|
+
|
|
175
|
+
# Imports the Google Cloud client library
|
|
176
|
+
from google.cloud import storage
|
|
177
|
+
|
|
178
|
+
# Instantiates a client
|
|
179
|
+
storage_client = storage.Client()
|
|
180
|
+
|
|
181
|
+
# The name for the new bucket
|
|
182
|
+
bucket_name = "my-new-bucket"
|
|
183
|
+
|
|
184
|
+
# Creates the new bucket
|
|
185
|
+
bucket = storage_client.create_bucket(bucket_name)
|
|
186
|
+
|
|
187
|
+
print(f"Bucket {bucket.name} created.")
|
|
188
|
+
|
|
189
|
+
|
|
224
190
|
Tracing With OpenTelemetry
|
|
225
191
|
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
226
192
|
|
|
@@ -275,6 +241,77 @@ For a list of libraries that can be instrumented, refer to the `OpenTelemetry Re
|
|
|
275
241
|
.. _Google Cloud Trace: https://cloud.google.com/trace
|
|
276
242
|
|
|
277
243
|
|
|
244
|
+
3.0 Major Version Notes
|
|
245
|
+
-----------------------
|
|
246
|
+
|
|
247
|
+
Feedback Welcome
|
|
248
|
+
~~~~~~~~~~~~~~~~
|
|
249
|
+
|
|
250
|
+
If you experience that backwards compatibility for your application is broken
|
|
251
|
+
with this major version release, please let us know through the Github issues
|
|
252
|
+
system. While some breaks of backwards compatibility may be unavoidable due to
|
|
253
|
+
new features in the major version release, we will do our best to minimize
|
|
254
|
+
them. Thank you.
|
|
255
|
+
|
|
256
|
+
Exception Handling
|
|
257
|
+
~~~~~~~~~~~~~~~~~~
|
|
258
|
+
|
|
259
|
+
In Python Storage 3.0, the dependency ``google-resumable-media`` was integrated.
|
|
260
|
+
The ``google-resumable-media`` dependency included exceptions
|
|
261
|
+
``google.resumable_media.common.InvalidResponse`` and
|
|
262
|
+
``google.resumable_media.common.DataCorruption``, which were often imported
|
|
263
|
+
directly in user application code. The replacements for these exceptions are
|
|
264
|
+
``google.cloud.storage.exceptions.InvalidResponse`` and
|
|
265
|
+
``google.cloud.storage.exceptions.DataCorruption``. Please update application code
|
|
266
|
+
to import and use these exceptions instead.
|
|
267
|
+
|
|
268
|
+
For backwards compatibility, if ``google-resumable-media`` is installed, the new
|
|
269
|
+
exceptions will be defined as subclasses of the old exceptions, so applications
|
|
270
|
+
should continue to work without modification. This backwards compatibility
|
|
271
|
+
feature may be removed in a future major version update.
|
|
272
|
+
|
|
273
|
+
Some users may be using the original exception classes from the
|
|
274
|
+
``google-resumable-media`` library without explicitly installing that library. So
|
|
275
|
+
as not to break user applications following this pattern,
|
|
276
|
+
``google-resumable-media`` is still in the list of dependencies in this package's
|
|
277
|
+
setup.py file. Applications which do not import directly from
|
|
278
|
+
``google-resumable-media`` can safely disregard this dependency.
|
|
279
|
+
This backwards compatibility feature **will be removed** in a future major
|
|
280
|
+
version update. Please migrate to using the ``google.cloud.storage.exceptions``
|
|
281
|
+
classes as above.
|
|
282
|
+
|
|
283
|
+
Checksum Defaults
|
|
284
|
+
~~~~~~~~~~~~~~~~~
|
|
285
|
+
|
|
286
|
+
In Python Storage 3.0, uploads and downloads now have a default of "auto" where
|
|
287
|
+
applicable. "Auto" will use crc32c checksums, except for unusual cases where the
|
|
288
|
+
fast (C extension) crc32c implementation is not available, in which case it will
|
|
289
|
+
use md5 instead. Before Python Storage 3.0, the default was md5 for most
|
|
290
|
+
downloads and None for most uploads. Note that ranged downloads ("start" or
|
|
291
|
+
"end" set) still do not support any checksumming, and some features in
|
|
292
|
+
``transfer_manager.py`` still support crc32c only.
|
|
293
|
+
|
|
294
|
+
Note: The method ``Blob.upload_from_file()`` requires a file in bytes mode, but
|
|
295
|
+
when checksum is set to None, as was the previous default, would not throw an
|
|
296
|
+
error if passed a file in string mode under some circumstances. With the new
|
|
297
|
+
defaults, it will now raise a TypeError. Please use a file opened in bytes
|
|
298
|
+
reading mode as required.
|
|
299
|
+
|
|
300
|
+
Miscellaneous
|
|
301
|
+
~~~~~~~~~~~~~
|
|
302
|
+
|
|
303
|
+
- The ``BlobWriter`` class now attempts to terminate an ongoing resumable upload if
|
|
304
|
+
the writer exits with an exception.
|
|
305
|
+
- Retry behavior is now identical between media operations (uploads and
|
|
306
|
+
downloads) and other operations, and custom predicates are now supported for
|
|
307
|
+
media operations as well.
|
|
308
|
+
- ``Blob.download_as_filename()`` will now delete the empty file if it results in a
|
|
309
|
+
google.cloud.exceptions.NotFound exception (HTTP 404).
|
|
310
|
+
- Previously, object upload, metadata update, and delete methods had retries
|
|
311
|
+
disabled by default unless the generation or metageneration was specified in
|
|
312
|
+
the request. This has now changed so that retries are enabled by default.
|
|
313
|
+
|
|
314
|
+
|
|
278
315
|
Next Steps
|
|
279
316
|
~~~~~~~~~~
|
|
280
317
|
|
|
@@ -8,6 +8,8 @@ allows world-wide storage and retrieval of any amount of data at any time. You c
|
|
|
8
8
|
Cloud Storage for a range of scenarios including serving website content, storing data
|
|
9
9
|
for archival and disaster recovery, or distributing large data objects to users via direct download.
|
|
10
10
|
|
|
11
|
+
**NOTE**: `3.0 Major Version Notes`_ are available. Feedback welcome.
|
|
12
|
+
|
|
11
13
|
A comprehensive list of changes in each version may be found in the `CHANGELOG`_.
|
|
12
14
|
|
|
13
15
|
- `Product Documentation`_
|
|
@@ -29,7 +31,7 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
|
|
|
29
31
|
.. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-storage.svg
|
|
30
32
|
:target: https://pypi.org/project/google-cloud-storage/
|
|
31
33
|
.. _Google Cloud Storage: https://cloud.google.com/storage
|
|
32
|
-
.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/storage/latest
|
|
34
|
+
.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/storage/latest/summary_overview
|
|
33
35
|
.. _Product Documentation: https://cloud.google.com/storage
|
|
34
36
|
.. _CHANGELOG: https://github.com/googleapis/python-storage/blob/main/CHANGELOG.md
|
|
35
37
|
.. _github.com/googleapis/python-storage: https://github.com/googleapis/python-storage
|
|
@@ -37,75 +39,6 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
|
|
|
37
39
|
.. _Storage Control API: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2
|
|
38
40
|
.. _Client Libraries Explained: https://cloud.google.com/apis/docs/client-libraries-explained
|
|
39
41
|
|
|
40
|
-
3.0 Major Version Notes
|
|
41
|
-
-----------------------
|
|
42
|
-
|
|
43
|
-
Feedback Welcome
|
|
44
|
-
~~~~~~~~~~~~~~~~
|
|
45
|
-
|
|
46
|
-
If you experience that backwards compatibility for your application is broken
|
|
47
|
-
with this major version release, please let us know through the Github issues
|
|
48
|
-
system. While some breaks of backwards compatibility may be unavoidable due to
|
|
49
|
-
new features in the major version release, we will do our best to minimize
|
|
50
|
-
them. Thank you.
|
|
51
|
-
|
|
52
|
-
Exception Handling
|
|
53
|
-
~~~~~~~~~~~~~~~~~~
|
|
54
|
-
|
|
55
|
-
In Python Storage 3.0, the dependency ``google-resumable-media`` was integrated.
|
|
56
|
-
The ``google-resumable-media`` dependency included exceptions
|
|
57
|
-
``google.resumable_media.common.InvalidResponse`` and
|
|
58
|
-
``google.resumable_media.common.DataCorruption``, which were often imported
|
|
59
|
-
directly in user application code. The replacements for these exceptions are
|
|
60
|
-
``google.cloud.storage.exceptions.InvalidResponse`` and
|
|
61
|
-
``google.cloud.storage.exceptions.DataCorruption``. Please update application code
|
|
62
|
-
to import and use these exceptions instead.
|
|
63
|
-
|
|
64
|
-
For backwards compatibility, if ``google-resumable-media`` is installed, the new
|
|
65
|
-
exceptions will be defined as subclasses of the old exceptions, so applications
|
|
66
|
-
should continue to work without modification. This backwards compatibility
|
|
67
|
-
feature may be removed in a future major version update.
|
|
68
|
-
|
|
69
|
-
Some users may be using the original exception classes from the
|
|
70
|
-
``google-resumable-media`` library without explicitly installing that library. So
|
|
71
|
-
as not to break user applications following this pattern,
|
|
72
|
-
``google-resumable-media`` is still in the list of dependencies in this package's
|
|
73
|
-
setup.py file. Applications which do not import directly from
|
|
74
|
-
``google-resumable-media`` can safely disregard this dependency.
|
|
75
|
-
This backwards compatibility feature **will be removed** in a future major
|
|
76
|
-
version update. Please migrate to using the ``google.cloud.storage.exceptions``
|
|
77
|
-
classes as above.
|
|
78
|
-
|
|
79
|
-
Checksum Defaults
|
|
80
|
-
~~~~~~~~~~~~~~~~~
|
|
81
|
-
|
|
82
|
-
In Python Storage 3.0, uploads and downloads now have a default of "auto" where
|
|
83
|
-
applicable. "Auto" will use crc32c checksums, except for unusual cases where the
|
|
84
|
-
fast (C extension) crc32c implementation is not available, in which case it will
|
|
85
|
-
use md5 instead. Before Python Storage 3.0, the default was md5 for most
|
|
86
|
-
downloads and None for most uploads. Note that ranged downloads ("start" or
|
|
87
|
-
"end" set) still do not support any checksumming, and some features in
|
|
88
|
-
``transfer_manager.py`` still support crc32c only.
|
|
89
|
-
|
|
90
|
-
Note: The method ``Blob.upload_from_file()`` requires a file in bytes mode, but
|
|
91
|
-
when checksum is set to None, as was the previous default, would not throw an
|
|
92
|
-
error if passed a file in string mode under some circumstances. With the new
|
|
93
|
-
defaults, it will now raise a TypeError. Please use a file opened in bytes
|
|
94
|
-
reading mode as required.
|
|
95
|
-
|
|
96
|
-
Miscellaneous
|
|
97
|
-
~~~~~~~~~~~~~
|
|
98
|
-
|
|
99
|
-
- The ``BlobWriter`` class now attempts to terminate an ongoing resumable upload if
|
|
100
|
-
the writer exits with an exception.
|
|
101
|
-
- Retry behavior is now identical between media operations (uploads and
|
|
102
|
-
downloads) and other operations, and custom predicates are now supported for
|
|
103
|
-
media operations as well.
|
|
104
|
-
- ``Blob.download_as_filename()`` will now delete the empty file if it results in a
|
|
105
|
-
google.cloud.exceptions.NotFound exception (HTTP 404).
|
|
106
|
-
- Previously, object upload, metadata update, and delete methods had retries
|
|
107
|
-
disabled by default unless the generation or metageneration was specified in
|
|
108
|
-
the request. This has now changed so that retries are enabled by default.
|
|
109
42
|
|
|
110
43
|
Quick Start
|
|
111
44
|
-----------
|
|
@@ -186,6 +119,26 @@ Windows
|
|
|
186
119
|
pip install google-cloud-storage
|
|
187
120
|
|
|
188
121
|
|
|
122
|
+
Example Usage
|
|
123
|
+
~~~~~~~~~~~~~
|
|
124
|
+
|
|
125
|
+
.. code-block:: python
|
|
126
|
+
|
|
127
|
+
# Imports the Google Cloud client library
|
|
128
|
+
from google.cloud import storage
|
|
129
|
+
|
|
130
|
+
# Instantiates a client
|
|
131
|
+
storage_client = storage.Client()
|
|
132
|
+
|
|
133
|
+
# The name for the new bucket
|
|
134
|
+
bucket_name = "my-new-bucket"
|
|
135
|
+
|
|
136
|
+
# Creates the new bucket
|
|
137
|
+
bucket = storage_client.create_bucket(bucket_name)
|
|
138
|
+
|
|
139
|
+
print(f"Bucket {bucket.name} created.")
|
|
140
|
+
|
|
141
|
+
|
|
189
142
|
Tracing With OpenTelemetry
|
|
190
143
|
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
191
144
|
|
|
@@ -240,6 +193,77 @@ For a list of libraries that can be instrumented, refer to the `OpenTelemetry Re
|
|
|
240
193
|
.. _Google Cloud Trace: https://cloud.google.com/trace
|
|
241
194
|
|
|
242
195
|
|
|
196
|
+
3.0 Major Version Notes
|
|
197
|
+
-----------------------
|
|
198
|
+
|
|
199
|
+
Feedback Welcome
|
|
200
|
+
~~~~~~~~~~~~~~~~
|
|
201
|
+
|
|
202
|
+
If you experience that backwards compatibility for your application is broken
|
|
203
|
+
with this major version release, please let us know through the Github issues
|
|
204
|
+
system. While some breaks of backwards compatibility may be unavoidable due to
|
|
205
|
+
new features in the major version release, we will do our best to minimize
|
|
206
|
+
them. Thank you.
|
|
207
|
+
|
|
208
|
+
Exception Handling
|
|
209
|
+
~~~~~~~~~~~~~~~~~~
|
|
210
|
+
|
|
211
|
+
In Python Storage 3.0, the dependency ``google-resumable-media`` was integrated.
|
|
212
|
+
The ``google-resumable-media`` dependency included exceptions
|
|
213
|
+
``google.resumable_media.common.InvalidResponse`` and
|
|
214
|
+
``google.resumable_media.common.DataCorruption``, which were often imported
|
|
215
|
+
directly in user application code. The replacements for these exceptions are
|
|
216
|
+
``google.cloud.storage.exceptions.InvalidResponse`` and
|
|
217
|
+
``google.cloud.storage.exceptions.DataCorruption``. Please update application code
|
|
218
|
+
to import and use these exceptions instead.
|
|
219
|
+
|
|
220
|
+
For backwards compatibility, if ``google-resumable-media`` is installed, the new
|
|
221
|
+
exceptions will be defined as subclasses of the old exceptions, so applications
|
|
222
|
+
should continue to work without modification. This backwards compatibility
|
|
223
|
+
feature may be removed in a future major version update.
|
|
224
|
+
|
|
225
|
+
Some users may be using the original exception classes from the
|
|
226
|
+
``google-resumable-media`` library without explicitly installing that library. So
|
|
227
|
+
as not to break user applications following this pattern,
|
|
228
|
+
``google-resumable-media`` is still in the list of dependencies in this package's
|
|
229
|
+
setup.py file. Applications which do not import directly from
|
|
230
|
+
``google-resumable-media`` can safely disregard this dependency.
|
|
231
|
+
This backwards compatibility feature **will be removed** in a future major
|
|
232
|
+
version update. Please migrate to using the ``google.cloud.storage.exceptions``
|
|
233
|
+
classes as above.
|
|
234
|
+
|
|
235
|
+
Checksum Defaults
|
|
236
|
+
~~~~~~~~~~~~~~~~~
|
|
237
|
+
|
|
238
|
+
In Python Storage 3.0, uploads and downloads now have a default of "auto" where
|
|
239
|
+
applicable. "Auto" will use crc32c checksums, except for unusual cases where the
|
|
240
|
+
fast (C extension) crc32c implementation is not available, in which case it will
|
|
241
|
+
use md5 instead. Before Python Storage 3.0, the default was md5 for most
|
|
242
|
+
downloads and None for most uploads. Note that ranged downloads ("start" or
|
|
243
|
+
"end" set) still do not support any checksumming, and some features in
|
|
244
|
+
``transfer_manager.py`` still support crc32c only.
|
|
245
|
+
|
|
246
|
+
Note: The method ``Blob.upload_from_file()`` requires a file in bytes mode, but
|
|
247
|
+
when checksum is set to None, as was the previous default, would not throw an
|
|
248
|
+
error if passed a file in string mode under some circumstances. With the new
|
|
249
|
+
defaults, it will now raise a TypeError. Please use a file opened in bytes
|
|
250
|
+
reading mode as required.
|
|
251
|
+
|
|
252
|
+
Miscellaneous
|
|
253
|
+
~~~~~~~~~~~~~
|
|
254
|
+
|
|
255
|
+
- The ``BlobWriter`` class now attempts to terminate an ongoing resumable upload if
|
|
256
|
+
the writer exits with an exception.
|
|
257
|
+
- Retry behavior is now identical between media operations (uploads and
|
|
258
|
+
downloads) and other operations, and custom predicates are now supported for
|
|
259
|
+
media operations as well.
|
|
260
|
+
- ``Blob.download_as_filename()`` will now delete the empty file if it results in a
|
|
261
|
+
google.cloud.exceptions.NotFound exception (HTTP 404).
|
|
262
|
+
- Previously, object upload, metadata update, and delete methods had retries
|
|
263
|
+
disabled by default unless the generation or metageneration was specified in
|
|
264
|
+
the request. This has now changed so that retries are enabled by default.
|
|
265
|
+
|
|
266
|
+
|
|
243
267
|
Next Steps
|
|
244
268
|
~~~~~~~~~~
|
|
245
269
|
|
{google_cloud_storage-3.1.0 → google_cloud_storage-3.2.0}/google/cloud/storage/_media/_download.py
RENAMED
|
@@ -140,7 +140,7 @@ class Download(DownloadBase):
|
|
|
140
140
|
``start`` to the end of the media.
|
|
141
141
|
headers (Optional[Mapping[str, str]]): Extra headers that should
|
|
142
142
|
be sent with the request, e.g. headers for encrypted data.
|
|
143
|
-
checksum Optional
|
|
143
|
+
checksum (Optional[str]): The type of checksum to compute to verify
|
|
144
144
|
the integrity of the object. The response headers must contain
|
|
145
145
|
a checksum of the requested type. If the headers lack an
|
|
146
146
|
appropriate checksum (for instance in the case of transcoded or
|
|
@@ -157,6 +157,9 @@ class Download(DownloadBase):
|
|
|
157
157
|
See the retry.py source code and docstrings in this package
|
|
158
158
|
(google.cloud.storage.retry) for information on retry types and how
|
|
159
159
|
to configure them.
|
|
160
|
+
single_shot_download (Optional[bool]): If true, download the object in a single request.
|
|
161
|
+
Caution: Enabling this will increase the memory overload for your application.
|
|
162
|
+
Please enable this as per your use case.
|
|
160
163
|
|
|
161
164
|
"""
|
|
162
165
|
|
|
@@ -169,6 +172,7 @@ class Download(DownloadBase):
|
|
|
169
172
|
headers=None,
|
|
170
173
|
checksum="auto",
|
|
171
174
|
retry=DEFAULT_RETRY,
|
|
175
|
+
single_shot_download=False,
|
|
172
176
|
):
|
|
173
177
|
super(Download, self).__init__(
|
|
174
178
|
media_url, stream=stream, start=start, end=end, headers=headers, retry=retry
|
|
@@ -178,6 +182,7 @@ class Download(DownloadBase):
|
|
|
178
182
|
self.checksum = (
|
|
179
183
|
"crc32c" if _helpers._is_crc32c_available_and_fast() else "md5"
|
|
180
184
|
)
|
|
185
|
+
self.single_shot_download = single_shot_download
|
|
181
186
|
self._bytes_downloaded = 0
|
|
182
187
|
self._expected_checksum = None
|
|
183
188
|
self._checksum_object = None
|
|
@@ -43,6 +43,12 @@ Error writing to stream while handling a gzip-compressed file download.
|
|
|
43
43
|
Please restart the download.
|
|
44
44
|
"""
|
|
45
45
|
|
|
46
|
+
_RESPONSE_HEADERS_INFO = """\
|
|
47
|
+
The X-Goog-Stored-Content-Length is {}. The X-Goog-Stored-Content-Encoding is {}.
|
|
48
|
+
The download request read {} bytes of data.
|
|
49
|
+
If the download was incomplete, please check the network connection and restart the download.
|
|
50
|
+
"""
|
|
51
|
+
|
|
46
52
|
|
|
47
53
|
class Download(_request_helpers.RequestsMixin, _download.Download):
|
|
48
54
|
"""Helper to manage downloading a resource from a Google API.
|
|
@@ -126,13 +132,24 @@ class Download(_request_helpers.RequestsMixin, _download.Download):
|
|
|
126
132
|
# the stream is indeed compressed, this will delegate the checksum
|
|
127
133
|
# object to the decoder and return a _DoNothingHash here.
|
|
128
134
|
local_checksum_object = _add_decoder(response.raw, checksum_object)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
self.
|
|
135
|
-
|
|
135
|
+
|
|
136
|
+
# This is useful for smaller files, or when the user wants to
|
|
137
|
+
# download the entire file in one go.
|
|
138
|
+
if self.single_shot_download:
|
|
139
|
+
content = response.raw.read(decode_content=True)
|
|
140
|
+
self._stream.write(content)
|
|
141
|
+
self._bytes_downloaded += len(content)
|
|
142
|
+
local_checksum_object.update(content)
|
|
143
|
+
response._content_consumed = True
|
|
144
|
+
else:
|
|
145
|
+
body_iter = response.iter_content(
|
|
146
|
+
chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE,
|
|
147
|
+
decode_unicode=False,
|
|
148
|
+
)
|
|
149
|
+
for chunk in body_iter:
|
|
150
|
+
self._stream.write(chunk)
|
|
151
|
+
self._bytes_downloaded += len(chunk)
|
|
152
|
+
local_checksum_object.update(chunk)
|
|
136
153
|
|
|
137
154
|
# Don't validate the checksum for partial responses.
|
|
138
155
|
if (
|
|
@@ -141,13 +158,30 @@ class Download(_request_helpers.RequestsMixin, _download.Download):
|
|
|
141
158
|
):
|
|
142
159
|
actual_checksum = _helpers.prepare_checksum_digest(checksum_object.digest())
|
|
143
160
|
if actual_checksum != expected_checksum:
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
161
|
+
headers = self._get_headers(response)
|
|
162
|
+
x_goog_encoding = headers.get("x-goog-stored-content-encoding")
|
|
163
|
+
x_goog_length = headers.get("x-goog-stored-content-length")
|
|
164
|
+
content_length_msg = _RESPONSE_HEADERS_INFO.format(
|
|
165
|
+
x_goog_length, x_goog_encoding, self._bytes_downloaded
|
|
149
166
|
)
|
|
150
|
-
|
|
167
|
+
if (
|
|
168
|
+
x_goog_length
|
|
169
|
+
and self._bytes_downloaded < int(x_goog_length)
|
|
170
|
+
and x_goog_encoding != "gzip"
|
|
171
|
+
):
|
|
172
|
+
# The library will attempt to trigger a retry by raising a ConnectionError, if
|
|
173
|
+
# (a) bytes_downloaded is less than response header x-goog-stored-content-length, and
|
|
174
|
+
# (b) the object is not gzip-compressed when stored in Cloud Storage.
|
|
175
|
+
raise ConnectionError(content_length_msg)
|
|
176
|
+
else:
|
|
177
|
+
msg = _CHECKSUM_MISMATCH.format(
|
|
178
|
+
self.media_url,
|
|
179
|
+
expected_checksum,
|
|
180
|
+
actual_checksum,
|
|
181
|
+
checksum_type=self.checksum.upper(),
|
|
182
|
+
)
|
|
183
|
+
msg += content_length_msg
|
|
184
|
+
raise DataCorruption(response, msg)
|
|
151
185
|
|
|
152
186
|
def consume(
|
|
153
187
|
self,
|
|
@@ -322,13 +356,21 @@ class RawDownload(_request_helpers.RawRequestsMixin, _download.Download):
|
|
|
322
356
|
checksum_object = self._checksum_object
|
|
323
357
|
|
|
324
358
|
with response:
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
self._stream.write(
|
|
330
|
-
self._bytes_downloaded += len(
|
|
331
|
-
checksum_object.update(
|
|
359
|
+
# This is useful for smaller files, or when the user wants to
|
|
360
|
+
# download the entire file in one go.
|
|
361
|
+
if self.single_shot_download:
|
|
362
|
+
content = response.raw.read()
|
|
363
|
+
self._stream.write(content)
|
|
364
|
+
self._bytes_downloaded += len(content)
|
|
365
|
+
checksum_object.update(content)
|
|
366
|
+
else:
|
|
367
|
+
body_iter = response.raw.stream(
|
|
368
|
+
_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
|
|
369
|
+
)
|
|
370
|
+
for chunk in body_iter:
|
|
371
|
+
self._stream.write(chunk)
|
|
372
|
+
self._bytes_downloaded += len(chunk)
|
|
373
|
+
checksum_object.update(chunk)
|
|
332
374
|
response._content_consumed = True
|
|
333
375
|
|
|
334
376
|
# Don't validate the checksum for partial responses.
|
|
@@ -339,13 +381,30 @@ class RawDownload(_request_helpers.RawRequestsMixin, _download.Download):
|
|
|
339
381
|
actual_checksum = _helpers.prepare_checksum_digest(checksum_object.digest())
|
|
340
382
|
|
|
341
383
|
if actual_checksum != expected_checksum:
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
384
|
+
headers = self._get_headers(response)
|
|
385
|
+
x_goog_encoding = headers.get("x-goog-stored-content-encoding")
|
|
386
|
+
x_goog_length = headers.get("x-goog-stored-content-length")
|
|
387
|
+
content_length_msg = _RESPONSE_HEADERS_INFO.format(
|
|
388
|
+
x_goog_length, x_goog_encoding, self._bytes_downloaded
|
|
347
389
|
)
|
|
348
|
-
|
|
390
|
+
if (
|
|
391
|
+
x_goog_length
|
|
392
|
+
and self._bytes_downloaded < int(x_goog_length)
|
|
393
|
+
and x_goog_encoding != "gzip"
|
|
394
|
+
):
|
|
395
|
+
# The library will attempt to trigger a retry by raising a ConnectionError, if
|
|
396
|
+
# (a) bytes_downloaded is less than response header x-goog-stored-content-length, and
|
|
397
|
+
# (b) the object is not gzip-compressed when stored in Cloud Storage.
|
|
398
|
+
raise ConnectionError(content_length_msg)
|
|
399
|
+
else:
|
|
400
|
+
msg = _CHECKSUM_MISMATCH.format(
|
|
401
|
+
self.media_url,
|
|
402
|
+
expected_checksum,
|
|
403
|
+
actual_checksum,
|
|
404
|
+
checksum_type=self.checksum.upper(),
|
|
405
|
+
)
|
|
406
|
+
msg += content_length_msg
|
|
407
|
+
raise DataCorruption(response, msg)
|
|
349
408
|
|
|
350
409
|
def consume(
|
|
351
410
|
self,
|