flask-storage 2.1.1.dev1__py3-none-any.whl → 2.1.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flask_storage/backends/s3.py +18 -9
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/METADATA +1 -1
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/RECORD +7 -7
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/WHEEL +0 -0
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/entry_points.txt +0 -0
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/licenses/LICENSE +0 -0
- {flask_storage-2.1.1.dev1.dist-info → flask_storage-2.1.2.dev1.dist-info}/top_level.txt +0 -0
flask_storage/backends/s3.py
CHANGED
|
@@ -50,6 +50,8 @@ class S3Backend(BaseBackend):
|
|
|
50
50
|
and optionally:
|
|
51
51
|
|
|
52
52
|
- `create_bucket`: Create the bucket at startup if it does not exist
|
|
53
|
+
- `use_threads`: Whether a multipart upload may send its parts in parallel
|
|
54
|
+
from a pool of threads. ``False`` by default.
|
|
53
55
|
"""
|
|
54
56
|
|
|
55
57
|
_S3_OBJECT_OPTION_MAP = {
|
|
@@ -130,21 +132,28 @@ class S3Backend(BaseBackend):
|
|
|
130
132
|
# digests. `FULL_OBJECT` asks for a digest of the content instead, so a
|
|
131
133
|
# file has the same checksum however many parts it travelled in.
|
|
132
134
|
#
|
|
133
|
-
# `use_threads` is
|
|
134
|
-
#
|
|
135
|
-
#
|
|
136
|
-
#
|
|
137
|
-
#
|
|
138
|
-
#
|
|
139
|
-
#
|
|
140
|
-
#
|
|
135
|
+
# `use_threads` is configurable: `False` by default to preserve the
|
|
136
|
+
# historical behavior of a single sequential transfer. For a stream
|
|
137
|
+
# that cannot seek, s3transfer reads the parts one after the other in
|
|
138
|
+
# the single submission thread (each fully buffered in memory), then
|
|
139
|
+
# hands them to the pool's worker threads to be sent in parallel, so
|
|
140
|
+
# the stream is never read concurrently nor rewound and the checksum
|
|
141
|
+
# measured as it is read stays valid. Where the transfer is bound by
|
|
142
|
+
# round-trips rather than bandwidth (remote regions in particular),
|
|
143
|
+
# parallelizing the part uploads can speed the upload up severalfold:
|
|
144
|
+
# set `use_threads=True` in the storage config (e.g.
|
|
145
|
+
# `RESOURCES_FS_USE_THREADS`) to enable it. Hosts unable to afford the
|
|
146
|
+
# thread pool (a handful of threads per upload; a host short on memory
|
|
147
|
+
# to map their stacks fails the upload outright with "can't start new
|
|
148
|
+
# thread") are the reason it is not on by default.
|
|
141
149
|
extra_args = self.get_object_extra_args(filename)
|
|
142
150
|
extra_args["ChecksumType"] = "FULL_OBJECT"
|
|
151
|
+
use_threads = self.config.get("use_threads", False) or False
|
|
143
152
|
self.bucket.upload_fileobj(
|
|
144
153
|
NonClosingProxy(file_or_wfs),
|
|
145
154
|
filename,
|
|
146
155
|
ExtraArgs=extra_args,
|
|
147
|
-
Config=TransferConfig(use_threads=
|
|
156
|
+
Config=TransferConfig(use_threads=use_threads),
|
|
148
157
|
)
|
|
149
158
|
return filename
|
|
150
159
|
|
|
@@ -9,10 +9,10 @@ flask_storage/views.py,sha256=YngzSXuxwiT-GQFLNWrXk0SrPcyKmbrDCOubMt3Wo9g,325
|
|
|
9
9
|
flask_storage/backends/__init__.py,sha256=thajBIF5Ok5zAiLEZmYBHwTxjmtD6_-Y1kXnNYQH5ws,4051
|
|
10
10
|
flask_storage/backends/local.py,sha256=ooPnw2lZyX_sLgluK4sC1rQHaSBif7TPVwP0R66oNiM,4423
|
|
11
11
|
flask_storage/backends/mock.py,sha256=0-b6SfzIuVO5Uu8BlSz1uNNALyBGNHePz5o79Ef1aOw,185
|
|
12
|
-
flask_storage/backends/s3.py,sha256=
|
|
13
|
-
flask_storage-2.1.
|
|
14
|
-
flask_storage-2.1.
|
|
15
|
-
flask_storage-2.1.
|
|
16
|
-
flask_storage-2.1.
|
|
17
|
-
flask_storage-2.1.
|
|
18
|
-
flask_storage-2.1.
|
|
12
|
+
flask_storage/backends/s3.py,sha256=oT7BR1bPpbNpSmO3IPWzuLil1r0tuu00v-nSTjIq4qk,10972
|
|
13
|
+
flask_storage-2.1.2.dev1.dist-info/licenses/LICENSE,sha256=Jsa1CKm-fF0guY0ht3reKgv2D3ReoCjNVX0IPtreqSE,1081
|
|
14
|
+
flask_storage-2.1.2.dev1.dist-info/METADATA,sha256=F0osrnREDeuno53tC367kvz67JBYdzdYEjvnnDmfKFQ,1312
|
|
15
|
+
flask_storage-2.1.2.dev1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
16
|
+
flask_storage-2.1.2.dev1.dist-info/entry_points.txt,sha256=hX7fv5H16y_4DLa-TJKZrdSqcddfRo1AEFSN3oH0cHs,156
|
|
17
|
+
flask_storage-2.1.2.dev1.dist-info/top_level.txt,sha256=dewU5rlVVfXSOtwXxhCTSujH1TX9-8SXFK9L67Lux3U,14
|
|
18
|
+
flask_storage-2.1.2.dev1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|