flask-storage 2.1.1.dev1__py3-none-any.whl → 2.1.2.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -50,6 +50,8 @@ class S3Backend(BaseBackend):
50
50
  and optionally:
51
51
 
52
52
  - `create_bucket`: Create the bucket at startup if it does not exist
53
+ - `use_threads`: Whether a multipart upload may send its parts in parallel
54
+ from a pool of threads. ``False`` by default.
53
55
  """
54
56
 
55
57
  _S3_OBJECT_OPTION_MAP = {
@@ -130,21 +132,28 @@ class S3Backend(BaseBackend):
130
132
  # digests. `FULL_OBJECT` asks for a digest of the content instead, so a
131
133
  # file has the same checksum however many parts it travelled in.
132
134
  #
133
- # `use_threads` is off because a transfer otherwise builds a pool of a
134
- # dozen threads for itself: ten to make the requests, five to submit
135
- # them and one for the I/O. A caller serving a web request pays that per
136
- # upload and gains little, since the parts are read one after the other
137
- # from a single stream anyway, and a host short on memory to map their
138
- # stacks fails the upload outright with "can't start new thread". The
139
- # transfer is otherwise unchanged: only the executor differs, the
140
- # multipart logic and its bounded memory are the same.
135
+ # `use_threads` is configurable: `False` by default to preserve the
136
+ # historical behavior of a single sequential transfer. For a stream
137
+ # that cannot seek, s3transfer reads the parts one after the other in
138
+ # the single submission thread (each fully buffered in memory), then
139
+ # hands them to the pool's worker threads to be sent in parallel, so
140
+ # the stream is never read concurrently nor rewound and the checksum
141
+ # measured as it is read stays valid. Where the transfer is bound by
142
+ # round-trips rather than bandwidth (remote regions in particular),
143
+ # parallelizing the part uploads can speed the upload up severalfold:
144
+ # set `use_threads=True` in the storage config (e.g.
145
+ # `RESOURCES_FS_USE_THREADS`) to enable it. Hosts unable to afford the
146
+ # thread pool (a handful of threads per upload; a host short on memory
147
+ # to map their stacks fails the upload outright with "can't start new
148
+ # thread") are the reason it is not on by default.
141
149
  extra_args = self.get_object_extra_args(filename)
142
150
  extra_args["ChecksumType"] = "FULL_OBJECT"
151
+ use_threads = self.config.get("use_threads", False) or False
143
152
  self.bucket.upload_fileobj(
144
153
  NonClosingProxy(file_or_wfs),
145
154
  filename,
146
155
  ExtraArgs=extra_args,
147
- Config=TransferConfig(use_threads=False),
156
+ Config=TransferConfig(use_threads=use_threads),
148
157
  )
149
158
  return filename
150
159
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flask-storage
3
- Version: 2.1.1.dev1
3
+ Version: 2.1.2.dev1
4
4
  Summary: Simple and easy file storages for Flask
5
5
  Author-email: Etalab <etalab@data.gouv.fr>
6
6
  Project-URL: Home, https://github.com/etalab/flask-storage
@@ -9,10 +9,10 @@ flask_storage/views.py,sha256=YngzSXuxwiT-GQFLNWrXk0SrPcyKmbrDCOubMt3Wo9g,325
9
9
  flask_storage/backends/__init__.py,sha256=thajBIF5Ok5zAiLEZmYBHwTxjmtD6_-Y1kXnNYQH5ws,4051
10
10
  flask_storage/backends/local.py,sha256=ooPnw2lZyX_sLgluK4sC1rQHaSBif7TPVwP0R66oNiM,4423
11
11
  flask_storage/backends/mock.py,sha256=0-b6SfzIuVO5Uu8BlSz1uNNALyBGNHePz5o79Ef1aOw,185
12
- flask_storage/backends/s3.py,sha256=p_EBPjbgIbXZ1rjr7BNJL4HRNKxzzVCo5YCZKoaPhbs,10329
13
- flask_storage-2.1.1.dev1.dist-info/licenses/LICENSE,sha256=Jsa1CKm-fF0guY0ht3reKgv2D3ReoCjNVX0IPtreqSE,1081
14
- flask_storage-2.1.1.dev1.dist-info/METADATA,sha256=yzng19SsxSmuF2M2HFsP2Hj1ktCcADz7q02uxNgC7Pg,1312
15
- flask_storage-2.1.1.dev1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
16
- flask_storage-2.1.1.dev1.dist-info/entry_points.txt,sha256=hX7fv5H16y_4DLa-TJKZrdSqcddfRo1AEFSN3oH0cHs,156
17
- flask_storage-2.1.1.dev1.dist-info/top_level.txt,sha256=dewU5rlVVfXSOtwXxhCTSujH1TX9-8SXFK9L67Lux3U,14
18
- flask_storage-2.1.1.dev1.dist-info/RECORD,,
12
+ flask_storage/backends/s3.py,sha256=oT7BR1bPpbNpSmO3IPWzuLil1r0tuu00v-nSTjIq4qk,10972
13
+ flask_storage-2.1.2.dev1.dist-info/licenses/LICENSE,sha256=Jsa1CKm-fF0guY0ht3reKgv2D3ReoCjNVX0IPtreqSE,1081
14
+ flask_storage-2.1.2.dev1.dist-info/METADATA,sha256=F0osrnREDeuno53tC367kvz67JBYdzdYEjvnnDmfKFQ,1312
15
+ flask_storage-2.1.2.dev1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
16
+ flask_storage-2.1.2.dev1.dist-info/entry_points.txt,sha256=hX7fv5H16y_4DLa-TJKZrdSqcddfRo1AEFSN3oH0cHs,156
17
+ flask_storage-2.1.2.dev1.dist-info/top_level.txt,sha256=dewU5rlVVfXSOtwXxhCTSujH1TX9-8SXFK9L67Lux3U,14
18
+ flask_storage-2.1.2.dev1.dist-info/RECORD,,