swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/__init__.py +29 -50
- swift/account/auditor.py +21 -118
- swift/account/backend.py +33 -28
- swift/account/reaper.py +37 -28
- swift/account/replicator.py +22 -0
- swift/account/server.py +60 -26
- swift/account/utils.py +28 -11
- swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
- swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
- swift/cli/container_deleter.py +5 -11
- swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
- swift/cli/dispersion_report.py +10 -9
- swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
- swift/cli/form_signature.py +3 -7
- swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
- swift/cli/info.py +183 -29
- swift/cli/manage_shard_ranges.py +708 -37
- swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
- swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
- swift/cli/recon.py +196 -67
- swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
- swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +807 -126
- swift/cli/reload.py +135 -0
- swift/cli/ringbuilder.py +217 -20
- swift/cli/ringcomposer.py +0 -1
- swift/cli/shard-info.py +4 -3
- swift/common/base_storage_server.py +9 -20
- swift/common/bufferedhttp.py +48 -74
- swift/common/constraints.py +20 -15
- swift/common/container_sync_realms.py +9 -11
- swift/common/daemon.py +25 -8
- swift/common/db.py +198 -127
- swift/common/db_auditor.py +168 -0
- swift/common/db_replicator.py +95 -55
- swift/common/digest.py +141 -0
- swift/common/direct_client.py +144 -33
- swift/common/error_limiter.py +93 -0
- swift/common/exceptions.py +25 -1
- swift/common/header_key_dict.py +2 -9
- swift/common/http_protocol.py +373 -0
- swift/common/internal_client.py +129 -59
- swift/common/linkat.py +3 -4
- swift/common/manager.py +284 -67
- swift/common/memcached.py +396 -147
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +211 -46
- swift/common/middleware/acl.py +3 -8
- swift/common/middleware/backend_ratelimit.py +230 -0
- swift/common/middleware/bulk.py +22 -34
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +6 -11
- swift/common/middleware/container_quotas.py +1 -1
- swift/common/middleware/container_sync.py +39 -17
- swift/common/middleware/copy.py +12 -0
- swift/common/middleware/crossdomain.py +22 -9
- swift/common/middleware/crypto/__init__.py +2 -1
- swift/common/middleware/crypto/crypto_utils.py +11 -15
- swift/common/middleware/crypto/decrypter.py +28 -11
- swift/common/middleware/crypto/encrypter.py +12 -17
- swift/common/middleware/crypto/keymaster.py +8 -15
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/dlo.py +15 -11
- swift/common/middleware/domain_remap.py +5 -4
- swift/common/middleware/etag_quoter.py +128 -0
- swift/common/middleware/formpost.py +73 -70
- swift/common/middleware/gatekeeper.py +8 -1
- swift/common/middleware/keystoneauth.py +33 -3
- swift/common/middleware/list_endpoints.py +4 -4
- swift/common/middleware/listing_formats.py +85 -49
- swift/common/middleware/memcache.py +4 -81
- swift/common/middleware/name_check.py +3 -2
- swift/common/middleware/proxy_logging.py +160 -92
- swift/common/middleware/ratelimit.py +17 -10
- swift/common/middleware/read_only.py +6 -4
- swift/common/middleware/recon.py +59 -22
- swift/common/middleware/s3api/acl_handlers.py +25 -3
- swift/common/middleware/s3api/acl_utils.py +6 -1
- swift/common/middleware/s3api/controllers/__init__.py +6 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/bucket.py +242 -137
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
- swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
- swift/common/middleware/s3api/controllers/obj.py +112 -8
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
- swift/common/middleware/s3api/controllers/tagging.py +57 -0
- swift/common/middleware/s3api/controllers/versioning.py +36 -7
- swift/common/middleware/s3api/etree.py +22 -9
- swift/common/middleware/s3api/exception.py +0 -4
- swift/common/middleware/s3api/s3api.py +113 -41
- swift/common/middleware/s3api/s3request.py +384 -218
- swift/common/middleware/s3api/s3response.py +126 -23
- swift/common/middleware/s3api/s3token.py +16 -17
- swift/common/middleware/s3api/schema/delete.rng +1 -1
- swift/common/middleware/s3api/subresource.py +7 -10
- swift/common/middleware/s3api/utils.py +27 -10
- swift/common/middleware/slo.py +665 -358
- swift/common/middleware/staticweb.py +64 -37
- swift/common/middleware/symlink.py +52 -19
- swift/common/middleware/tempauth.py +76 -58
- swift/common/middleware/tempurl.py +192 -174
- swift/common/middleware/versioned_writes/__init__.py +51 -0
- swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
- swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +18 -19
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +10 -10
- swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
- swift/common/registry.py +147 -0
- swift/common/request_helpers.py +324 -57
- swift/common/ring/builder.py +67 -25
- swift/common/ring/composite_builder.py +1 -1
- swift/common/ring/ring.py +177 -51
- swift/common/ring/utils.py +1 -1
- swift/common/splice.py +10 -6
- swift/common/statsd_client.py +205 -0
- swift/common/storage_policy.py +49 -44
- swift/common/swob.py +86 -102
- swift/common/{utils.py → utils/__init__.py} +2191 -2762
- swift/common/utils/base.py +131 -0
- swift/common/utils/config.py +433 -0
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/logs.py +859 -0
- swift/common/utils/timestamp.py +412 -0
- swift/common/wsgi.py +555 -536
- swift/container/auditor.py +14 -100
- swift/container/backend.py +552 -227
- swift/container/reconciler.py +126 -37
- swift/container/replicator.py +96 -22
- swift/container/server.py +397 -176
- swift/container/sharder.py +1580 -639
- swift/container/sync.py +94 -88
- swift/container/updater.py +53 -32
- swift/obj/auditor.py +153 -35
- swift/obj/diskfile.py +466 -217
- swift/obj/expirer.py +406 -124
- swift/obj/mem_diskfile.py +7 -4
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +523 -262
- swift/obj/replicator.py +249 -188
- swift/obj/server.py +213 -122
- swift/obj/ssync_receiver.py +145 -85
- swift/obj/ssync_sender.py +113 -54
- swift/obj/updater.py +653 -139
- swift/obj/watchers/__init__.py +0 -0
- swift/obj/watchers/dark_data.py +213 -0
- swift/proxy/controllers/account.py +11 -11
- swift/proxy/controllers/base.py +848 -604
- swift/proxy/controllers/container.py +452 -86
- swift/proxy/controllers/info.py +3 -2
- swift/proxy/controllers/obj.py +1009 -490
- swift/proxy/server.py +185 -112
- swift-2.35.0.dist-info/AUTHORS +501 -0
- swift-2.35.0.dist-info/LICENSE +202 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
- swift-2.35.0.dist-info/RECORD +201 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
- swift-2.35.0.dist-info/pbr.json +1 -0
- swift/locale/de/LC_MESSAGES/swift.po +0 -1216
- swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
- swift/locale/es/LC_MESSAGES/swift.po +0 -1085
- swift/locale/fr/LC_MESSAGES/swift.po +0 -909
- swift/locale/it/LC_MESSAGES/swift.po +0 -894
- swift/locale/ja/LC_MESSAGES/swift.po +0 -965
- swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
- swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
- swift/locale/ru/LC_MESSAGES/swift.po +0 -891
- swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
- swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
- swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
- swift-2.23.2.data/scripts/swift-account-auditor +0 -23
- swift-2.23.2.data/scripts/swift-account-info +0 -51
- swift-2.23.2.data/scripts/swift-account-reaper +0 -23
- swift-2.23.2.data/scripts/swift-account-replicator +0 -34
- swift-2.23.2.data/scripts/swift-account-server +0 -23
- swift-2.23.2.data/scripts/swift-container-auditor +0 -23
- swift-2.23.2.data/scripts/swift-container-info +0 -51
- swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
- swift-2.23.2.data/scripts/swift-container-replicator +0 -34
- swift-2.23.2.data/scripts/swift-container-sharder +0 -33
- swift-2.23.2.data/scripts/swift-container-sync +0 -23
- swift-2.23.2.data/scripts/swift-container-updater +0 -23
- swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
- swift-2.23.2.data/scripts/swift-form-signature +0 -20
- swift-2.23.2.data/scripts/swift-init +0 -119
- swift-2.23.2.data/scripts/swift-object-auditor +0 -29
- swift-2.23.2.data/scripts/swift-object-expirer +0 -33
- swift-2.23.2.data/scripts/swift-object-info +0 -60
- swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
- swift-2.23.2.data/scripts/swift-object-relinker +0 -41
- swift-2.23.2.data/scripts/swift-object-replicator +0 -37
- swift-2.23.2.data/scripts/swift-object-server +0 -27
- swift-2.23.2.data/scripts/swift-object-updater +0 -23
- swift-2.23.2.data/scripts/swift-proxy-server +0 -23
- swift-2.23.2.data/scripts/swift-recon +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.23.2.data/scripts/swift-ring-composer +0 -22
- swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
- swift-2.23.2.dist-info/RECORD +0 -220
- swift-2.23.2.dist-info/metadata.json +0 -1
- swift-2.23.2.dist-info/pbr.json +0 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -64,7 +64,7 @@ signature is generated using the HTTP method to allow (``GET``, ``PUT``,
|
|
64
64
|
the full path to the object, and the key set on the account.
|
65
65
|
|
66
66
|
The digest algorithm to be used may be configured by the operator. By default,
|
67
|
-
HMAC-
|
67
|
+
HMAC-SHA256 and HMAC-SHA512 are supported. Check the
|
68
68
|
``tempurl.allowed_digests`` entry in the cluster's capabilities response to
|
69
69
|
see which algorithms are supported by your deployment; see
|
70
70
|
:doc:`api/discoverability` for more information. On older clusters,
|
@@ -75,24 +75,25 @@ For example, here is code generating the signature for a ``GET`` for 60
|
|
75
75
|
seconds on ``/v1/AUTH_account/container/object``::
|
76
76
|
|
77
77
|
import hmac
|
78
|
-
from hashlib import
|
78
|
+
from hashlib import sha256
|
79
79
|
from time import time
|
80
80
|
method = 'GET'
|
81
81
|
expires = int(time() + 60)
|
82
82
|
path = '/v1/AUTH_account/container/object'
|
83
83
|
key = 'mykey'
|
84
84
|
hmac_body = '%s\n%s\n%s' % (method, expires, path)
|
85
|
-
sig = hmac.new(key, hmac_body,
|
85
|
+
sig = hmac.new(key, hmac_body, sha256).hexdigest()
|
86
86
|
|
87
87
|
Be certain to use the full path, from the ``/v1/`` onward.
|
88
88
|
|
89
89
|
Let's say ``sig`` ends up equaling
|
90
|
-
``
|
91
|
-
``
|
90
|
+
``732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b`` and
|
91
|
+
``expires`` ends up ``1512508563``. Then, for example, the website could
|
92
|
+
provide a link to::
|
92
93
|
|
93
94
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
94
|
-
temp_url_sig=
|
95
|
-
temp_url_expires=
|
95
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
96
|
+
temp_url_expires=1512508563
|
96
97
|
|
97
98
|
For longer hashes, a hex encoding becomes unwieldy. Base64 encoding is also
|
98
99
|
supported, and indicated by prefixing the signature with ``"<digest name>:"``.
|
@@ -124,11 +125,11 @@ Supposing that ``sig`` ends up equaling
|
|
124
125
|
You may also use ISO 8601 UTC timestamps with the format
|
125
126
|
``"%Y-%m-%dT%H:%M:%SZ"`` instead of UNIX timestamps in the URL
|
126
127
|
(but NOT in the code above for generating the signature!).
|
127
|
-
So, the above HMAC-
|
128
|
+
So, the above HMAC-SHA246 URL could also be formulated as::
|
128
129
|
|
129
130
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
130
|
-
temp_url_sig=
|
131
|
-
temp_url_expires=
|
131
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
132
|
+
temp_url_expires=2017-12-05T21:16:03Z
|
132
133
|
|
133
134
|
If a prefix-based signature with the prefix ``pre`` is desired, set path to::
|
134
135
|
|
@@ -140,31 +141,31 @@ a query parameter called ``temp_url_prefix``. So, if ``sig`` and ``expires``
|
|
140
141
|
would end up like above, following URL would be valid::
|
141
142
|
|
142
143
|
https://swift-cluster.example.com/v1/AUTH_account/container/pre/object?
|
143
|
-
temp_url_sig=
|
144
|
-
temp_url_expires=
|
144
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
145
|
+
temp_url_expires=1512508563&
|
145
146
|
temp_url_prefix=pre
|
146
147
|
|
147
148
|
Another valid URL::
|
148
149
|
|
149
150
|
https://swift-cluster.example.com/v1/AUTH_account/container/pre/
|
150
151
|
subfolder/another_object?
|
151
|
-
temp_url_sig=
|
152
|
-
temp_url_expires=
|
152
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
153
|
+
temp_url_expires=1512508563&
|
153
154
|
temp_url_prefix=pre
|
154
155
|
|
155
156
|
If you wish to lock down the ip ranges from where the resource can be accessed
|
156
157
|
to the ip ``1.2.3.4``::
|
157
158
|
|
158
159
|
import hmac
|
159
|
-
from hashlib import
|
160
|
+
from hashlib import sha256
|
160
161
|
from time import time
|
161
162
|
method = 'GET'
|
162
163
|
expires = int(time() + 60)
|
163
164
|
path = '/v1/AUTH_account/container/object'
|
164
165
|
ip_range = '1.2.3.4'
|
165
|
-
key = 'mykey'
|
166
|
+
key = b'mykey'
|
166
167
|
hmac_body = 'ip=%s\n%s\n%s\n%s' % (ip_range, method, expires, path)
|
167
|
-
sig = hmac.new(key, hmac_body,
|
168
|
+
sig = hmac.new(key, hmac_body.encode('ascii'), sha256).hexdigest()
|
168
169
|
|
169
170
|
The generated signature would only be valid from the ip ``1.2.3.4``. The
|
170
171
|
middleware detects an ip-based temporary URL by a query parameter called
|
@@ -172,29 +173,29 @@ middleware detects an ip-based temporary URL by a query parameter called
|
|
172
173
|
above, following URL would be valid::
|
173
174
|
|
174
175
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
175
|
-
temp_url_sig=
|
176
|
-
temp_url_expires=
|
176
|
+
temp_url_sig=3f48476acaf5ec272acd8e99f7b5bad96c52ddba53ed27c60613711774a06f0c&
|
177
|
+
temp_url_expires=1648082711&
|
177
178
|
temp_url_ip_range=1.2.3.4
|
178
179
|
|
179
180
|
Similarly to lock down the ip to a range of ``1.2.3.X`` so starting
|
180
181
|
from the ip ``1.2.3.0`` to ``1.2.3.255``::
|
181
182
|
|
182
183
|
import hmac
|
183
|
-
from hashlib import
|
184
|
+
from hashlib import sha256
|
184
185
|
from time import time
|
185
186
|
method = 'GET'
|
186
187
|
expires = int(time() + 60)
|
187
188
|
path = '/v1/AUTH_account/container/object'
|
188
189
|
ip_range = '1.2.3.0/24'
|
189
|
-
key = 'mykey'
|
190
|
+
key = b'mykey'
|
190
191
|
hmac_body = 'ip=%s\n%s\n%s\n%s' % (ip_range, method, expires, path)
|
191
|
-
sig = hmac.new(key, hmac_body,
|
192
|
+
sig = hmac.new(key, hmac_body.encode('ascii'), sha256).hexdigest()
|
192
193
|
|
193
194
|
Then the following url would be valid::
|
194
195
|
|
195
196
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
196
|
-
temp_url_sig=
|
197
|
-
temp_url_expires=
|
197
|
+
temp_url_sig=6ff81256b8a3ba11d239da51a703b9c06a56ffddeb8caab74ca83af8f73c9c83&
|
198
|
+
temp_url_expires=1648082711&
|
198
199
|
temp_url_ip_range=1.2.3.0/24
|
199
200
|
|
200
201
|
|
@@ -222,16 +223,16 @@ can override this with a filename query parameter. Modifying the
|
|
222
223
|
above example::
|
223
224
|
|
224
225
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
225
|
-
temp_url_sig=
|
226
|
-
temp_url_expires=
|
226
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
227
|
+
temp_url_expires=1512508563&filename=My+Test+File.pdf
|
227
228
|
|
228
229
|
If you do not want the object to be downloaded, you can cause
|
229
230
|
``Content-Disposition: inline`` to be set on the response by adding the
|
230
231
|
``inline`` parameter to the query string, like so::
|
231
232
|
|
232
233
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
233
|
-
temp_url_sig=
|
234
|
-
temp_url_expires=
|
234
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
235
|
+
temp_url_expires=1512508563&inline
|
235
236
|
|
236
237
|
In some cases, the client might not able to present the content of the object,
|
237
238
|
but you still want the content able to save to local with the specific
|
@@ -240,8 +241,8 @@ set on the response by adding the ``inline&filename=...`` parameter to the
|
|
240
241
|
query string, like so::
|
241
242
|
|
242
243
|
https://swift-cluster.example.com/v1/AUTH_account/container/object?
|
243
|
-
temp_url_sig=
|
244
|
-
temp_url_expires=
|
244
|
+
temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
|
245
|
+
temp_url_expires=1512508563&inline&filename=My+Test+File.pdf
|
245
246
|
|
246
247
|
---------------------
|
247
248
|
Cluster Configuration
|
@@ -254,7 +255,7 @@ This middleware understands the following configuration settings:
|
|
254
255
|
incoming requests. Names may optionally end with ``*`` to
|
255
256
|
indicate a prefix match. ``incoming_allow_headers`` is a
|
256
257
|
list of exceptions to these removals.
|
257
|
-
Default: ``x-timestamp``
|
258
|
+
Default: ``x-timestamp x-open-expired``
|
258
259
|
|
259
260
|
``incoming_allow_headers``
|
260
261
|
A whitespace-delimited list of the headers allowed as
|
@@ -288,7 +289,7 @@ This middleware understands the following configuration settings:
|
|
288
289
|
A whitespace delimited list of digest algorithms that are allowed
|
289
290
|
to be used when calculating the signature for a temporary URL.
|
290
291
|
|
291
|
-
Default: ``
|
292
|
+
Default: ``sha256 sha512``
|
292
293
|
"""
|
293
294
|
|
294
295
|
__all__ = ['TempURL', 'filter_factory',
|
@@ -297,25 +298,24 @@ __all__ = ['TempURL', 'filter_factory',
|
|
297
298
|
'DEFAULT_OUTGOING_REMOVE_HEADERS',
|
298
299
|
'DEFAULT_OUTGOING_ALLOW_HEADERS']
|
299
300
|
|
300
|
-
import binascii
|
301
301
|
from calendar import timegm
|
302
|
-
import functools
|
303
|
-
import hashlib
|
304
|
-
import six
|
305
302
|
from os.path import basename
|
306
303
|
from time import time, strftime, strptime, gmtime
|
307
304
|
from ipaddress import ip_address, ip_network
|
308
305
|
|
309
|
-
from
|
310
|
-
from six.moves.urllib.parse import urlencode
|
306
|
+
from urllib.parse import parse_qs, urlencode
|
311
307
|
|
312
308
|
from swift.proxy.controllers.base import get_account_info, get_container_info
|
313
309
|
from swift.common.header_key_dict import HeaderKeyDict
|
310
|
+
from swift.common.http import is_success
|
311
|
+
from swift.common.digest import get_allowed_digests, \
|
312
|
+
extract_digest_and_algorithm, DEFAULT_ALLOWED_DIGESTS, get_hmac
|
314
313
|
from swift.common.swob import header_to_environ_key, HTTPUnauthorized, \
|
315
314
|
HTTPBadRequest, wsgi_to_str
|
316
|
-
from swift.common.utils import split_path,
|
317
|
-
|
318
|
-
|
315
|
+
from swift.common.utils import split_path, \
|
316
|
+
streq_const_time, quote, get_logger, close_if_possible
|
317
|
+
from swift.common.registry import register_swift_info, register_sensitive_param
|
318
|
+
from swift.common.wsgi import WSGIContext
|
319
319
|
|
320
320
|
|
321
321
|
DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target'
|
@@ -324,7 +324,7 @@ DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target'
|
|
324
324
|
#: delimited list of header names and names can optionally end with '*' to
|
325
325
|
#: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of
|
326
326
|
#: exceptions to these removals.
|
327
|
-
DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp'
|
327
|
+
DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp x-open-expired'
|
328
328
|
|
329
329
|
#: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a
|
330
330
|
#: whitespace delimited list of header names and names can optionally end with
|
@@ -342,9 +342,6 @@ DEFAULT_OUTGOING_REMOVE_HEADERS = 'x-object-meta-*'
|
|
342
342
|
#: '*' to indicate a prefix match.
|
343
343
|
DEFAULT_OUTGOING_ALLOW_HEADERS = 'x-object-meta-public-*'
|
344
344
|
|
345
|
-
DEFAULT_ALLOWED_DIGESTS = 'sha1 sha256 sha512'
|
346
|
-
SUPPORTED_DIGESTS = set(DEFAULT_ALLOWED_DIGESTS.split())
|
347
|
-
|
348
345
|
CONTAINER_SCOPE = 'container'
|
349
346
|
ACCOUNT_SCOPE = 'account'
|
350
347
|
|
@@ -362,11 +359,59 @@ def get_tempurl_keys_from_metadata(meta):
|
|
362
359
|
meta = get_account_info(...)['meta']
|
363
360
|
keys = get_tempurl_keys_from_metadata(meta)
|
364
361
|
"""
|
365
|
-
return [
|
366
|
-
for key, value in meta.items()
|
362
|
+
return [value for key, value in meta.items()
|
367
363
|
if key.lower() in ('temp-url-key', 'temp-url-key-2')]
|
368
364
|
|
369
365
|
|
366
|
+
def normalize_temp_url_expires(value):
|
367
|
+
"""
|
368
|
+
Returns the normalized expiration value as an int
|
369
|
+
|
370
|
+
If not None, the value is converted to an int if possible or 0
|
371
|
+
if not, and checked for expiration (returns 0 if expired).
|
372
|
+
"""
|
373
|
+
if value is None:
|
374
|
+
return value
|
375
|
+
try:
|
376
|
+
temp_url_expires = int(value)
|
377
|
+
except ValueError:
|
378
|
+
try:
|
379
|
+
temp_url_expires = timegm(strptime(
|
380
|
+
value, EXPIRES_ISO8601_FORMAT))
|
381
|
+
except ValueError:
|
382
|
+
temp_url_expires = 0
|
383
|
+
if temp_url_expires < time():
|
384
|
+
temp_url_expires = 0
|
385
|
+
return temp_url_expires
|
386
|
+
|
387
|
+
|
388
|
+
def get_temp_url_info(env):
|
389
|
+
"""
|
390
|
+
Returns the provided temporary URL parameters (sig, expires, prefix,
|
391
|
+
temp_url_ip_range), if given and syntactically valid.
|
392
|
+
Either sig, expires or prefix could be None if not provided.
|
393
|
+
|
394
|
+
:param env: The WSGI environment for the request.
|
395
|
+
:returns: (sig, expires, prefix, filename, inline,
|
396
|
+
temp_url_ip_range) as described above.
|
397
|
+
"""
|
398
|
+
sig = expires = prefix = ip_range = filename = inline = None
|
399
|
+
qs = parse_qs(env.get('QUERY_STRING', ''), keep_blank_values=True)
|
400
|
+
if 'temp_url_ip_range' in qs:
|
401
|
+
ip_range = qs['temp_url_ip_range'][0]
|
402
|
+
if 'temp_url_sig' in qs:
|
403
|
+
sig = qs['temp_url_sig'][0]
|
404
|
+
if 'temp_url_expires' in qs:
|
405
|
+
expires = qs['temp_url_expires'][0]
|
406
|
+
if 'temp_url_prefix' in qs:
|
407
|
+
prefix = qs['temp_url_prefix'][0]
|
408
|
+
if 'filename' in qs:
|
409
|
+
filename = qs['filename'][0]
|
410
|
+
if 'inline' in qs:
|
411
|
+
inline = True
|
412
|
+
return (sig, expires, prefix, filename, inline, ip_range)
|
413
|
+
|
414
|
+
|
370
415
|
def disposition_format(disposition_type, filename):
|
371
416
|
# Content-Disposition in HTTP is defined in
|
372
417
|
# https://tools.ietf.org/html/rfc6266 and references
|
@@ -424,11 +469,12 @@ class TempURL(object):
|
|
424
469
|
:param conf: The configuration dict for the middleware.
|
425
470
|
"""
|
426
471
|
|
427
|
-
def __init__(self, app, conf):
|
472
|
+
def __init__(self, app, conf, logger=None):
|
428
473
|
#: The next WSGI application/filter in the paste.deploy pipeline.
|
429
474
|
self.app = app
|
430
475
|
#: The filter configuration dict.
|
431
476
|
self.conf = conf
|
477
|
+
self.logger = logger or get_logger(conf, log_route='tempurl')
|
432
478
|
|
433
479
|
self.allowed_digests = conf.get(
|
434
480
|
'allowed_digests', DEFAULT_ALLOWED_DIGESTS.split())
|
@@ -497,36 +543,27 @@ class TempURL(object):
|
|
497
543
|
"""
|
498
544
|
if env['REQUEST_METHOD'] == 'OPTIONS':
|
499
545
|
return self.app(env, start_response)
|
500
|
-
info =
|
501
|
-
temp_url_sig,
|
546
|
+
info = get_temp_url_info(env)
|
547
|
+
temp_url_sig, client_temp_url_expires, temp_url_prefix, filename, \
|
502
548
|
inline_disposition, temp_url_ip_range = info
|
549
|
+
temp_url_expires = normalize_temp_url_expires(client_temp_url_expires)
|
503
550
|
if temp_url_sig is None and temp_url_expires is None:
|
504
551
|
return self.app(env, start_response)
|
505
552
|
if not temp_url_sig or not temp_url_expires:
|
506
553
|
return self._invalid(env, start_response)
|
507
554
|
|
508
|
-
|
509
|
-
hash_algorithm, temp_url_sig =
|
510
|
-
|
511
|
-
|
512
|
-
temp_url_sig = temp_url_sig.replace('-', '+').replace('_', '/')
|
513
|
-
try:
|
514
|
-
temp_url_sig = binascii.hexlify(strict_b64decode(
|
515
|
-
temp_url_sig + '=='))
|
516
|
-
if not six.PY2:
|
517
|
-
temp_url_sig = temp_url_sig.decode('ascii')
|
518
|
-
except ValueError:
|
519
|
-
return self._invalid(env, start_response)
|
520
|
-
elif len(temp_url_sig) == 40:
|
521
|
-
hash_algorithm = 'sha1'
|
522
|
-
elif len(temp_url_sig) == 64:
|
523
|
-
hash_algorithm = 'sha256'
|
524
|
-
else:
|
555
|
+
try:
|
556
|
+
hash_algorithm, temp_url_sig = extract_digest_and_algorithm(
|
557
|
+
temp_url_sig)
|
558
|
+
except ValueError:
|
525
559
|
return self._invalid(env, start_response)
|
526
560
|
if hash_algorithm not in self.allowed_digests:
|
527
561
|
return self._invalid(env, start_response)
|
528
562
|
|
529
|
-
account, container, obj = self._get_path_parts(
|
563
|
+
account, container, obj = self._get_path_parts(
|
564
|
+
env, allow_container_root=(
|
565
|
+
env['REQUEST_METHOD'] in ('GET', 'HEAD') and
|
566
|
+
temp_url_prefix == ""))
|
530
567
|
if not account:
|
531
568
|
return self._invalid(env, start_response)
|
532
569
|
|
@@ -535,8 +572,8 @@ class TempURL(object):
|
|
535
572
|
if client_address is None:
|
536
573
|
return self._invalid(env, start_response)
|
537
574
|
try:
|
538
|
-
allowed_ip_ranges = ip_network(
|
539
|
-
if ip_address(
|
575
|
+
allowed_ip_ranges = ip_network(str(temp_url_ip_range))
|
576
|
+
if ip_address(str(client_address)) not in allowed_ip_ranges:
|
540
577
|
return self._invalid(env, start_response)
|
541
578
|
except ValueError:
|
542
579
|
return self._invalid(env, start_response)
|
@@ -574,6 +611,7 @@ class TempURL(object):
|
|
574
611
|
break
|
575
612
|
if not is_valid_hmac:
|
576
613
|
return self._invalid(env, start_response)
|
614
|
+
self.logger.increment('tempurl.digests.%s' % hash_algorithm)
|
577
615
|
# disallowed headers prevent accidentally allowing upload of a pointer
|
578
616
|
# to data that the PUT tempurl would not otherwise allow access for.
|
579
617
|
# It should be safe to provide a GET tempurl for data that an
|
@@ -591,116 +629,102 @@ class TempURL(object):
|
|
591
629
|
env['swift.authorize_override'] = True
|
592
630
|
env['REMOTE_USER'] = '.wsgi.tempurl'
|
593
631
|
qs = {'temp_url_sig': temp_url_sig,
|
594
|
-
'temp_url_expires':
|
632
|
+
'temp_url_expires': client_temp_url_expires}
|
595
633
|
if temp_url_prefix is not None:
|
596
634
|
qs['temp_url_prefix'] = temp_url_prefix
|
597
635
|
if filename:
|
598
636
|
qs['filename'] = filename
|
599
637
|
env['QUERY_STRING'] = urlencode(qs)
|
600
638
|
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
if
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
639
|
+
ctx = WSGIContext(self.app)
|
640
|
+
app_iter = ctx._app_call(env)
|
641
|
+
ctx._response_headers = self._clean_outgoing_headers(
|
642
|
+
ctx._response_headers)
|
643
|
+
if env['REQUEST_METHOD'] in ('GET', 'HEAD') and \
|
644
|
+
is_success(ctx._get_status_int()):
|
645
|
+
# figure out the right value for content-disposition
|
646
|
+
# 1) use the value from the query string
|
647
|
+
# 2) use the value from the object metadata
|
648
|
+
# 3) use the object name (default)
|
649
|
+
out_headers = []
|
650
|
+
existing_disposition = None
|
651
|
+
content_generator = None
|
652
|
+
for h, v in ctx._response_headers:
|
653
|
+
if h.lower() == 'x-backend-content-generator':
|
654
|
+
content_generator = v
|
655
|
+
|
656
|
+
if h.lower() != 'content-disposition':
|
657
|
+
out_headers.append((h, v))
|
658
|
+
else:
|
659
|
+
existing_disposition = v
|
660
|
+
if content_generator == 'staticweb':
|
661
|
+
inline_disposition = True
|
662
|
+
elif obj == "":
|
663
|
+
# Generally, tempurl requires an object. We carved out an
|
664
|
+
# exception to allow GETs at the container root for the sake
|
665
|
+
# of staticweb, but we can't tell whether we'll have a
|
666
|
+
# staticweb response or not until after we call the app
|
667
|
+
close_if_possible(app_iter)
|
668
|
+
return self._invalid(env, start_response)
|
669
|
+
|
670
|
+
if inline_disposition:
|
671
|
+
if filename:
|
672
|
+
disposition_value = disposition_format('inline',
|
623
673
|
filename)
|
624
|
-
elif existing_disposition:
|
625
|
-
disposition_value = existing_disposition
|
626
674
|
else:
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
675
|
+
disposition_value = 'inline'
|
676
|
+
elif filename:
|
677
|
+
disposition_value = disposition_format('attachment',
|
678
|
+
filename)
|
679
|
+
elif existing_disposition:
|
680
|
+
disposition_value = existing_disposition
|
681
|
+
else:
|
682
|
+
name = basename(wsgi_to_str(env['PATH_INFO']).rstrip('/'))
|
683
|
+
disposition_value = disposition_format('attachment',
|
684
|
+
name)
|
685
|
+
# this is probably just paranoia, I couldn't actually get a
|
686
|
+
# newline into existing_disposition
|
687
|
+
value = disposition_value.replace('\n', '%0A')
|
688
|
+
out_headers.append(('Content-Disposition', value))
|
689
|
+
|
690
|
+
# include Expires header for better cache-control
|
691
|
+
out_headers.append(('Expires', strftime(
|
692
|
+
"%a, %d %b %Y %H:%M:%S GMT",
|
693
|
+
gmtime(temp_url_expires))))
|
694
|
+
ctx._response_headers = out_headers
|
695
|
+
start_response(
|
696
|
+
ctx._response_status,
|
697
|
+
ctx._response_headers,
|
698
|
+
ctx._response_exc_info)
|
699
|
+
return app_iter
|
700
|
+
|
701
|
+
def _get_path_parts(self, env, allow_container_root=False):
|
645
702
|
"""
|
646
703
|
Return the account, container and object name for the request,
|
647
704
|
if it's an object request and one of the configured methods;
|
648
705
|
otherwise, None is returned.
|
649
706
|
|
707
|
+
If it's a container request and allow_root_container is true,
|
708
|
+
the object name returned will be the empty string.
|
709
|
+
|
650
710
|
:param env: The WSGI environment for the request.
|
711
|
+
:param allow_container_root: Whether requests to the root of a
|
712
|
+
container should be allowed.
|
651
713
|
:returns: (Account str, container str, object str) or
|
652
714
|
(None, None, None).
|
653
715
|
"""
|
654
716
|
if env['REQUEST_METHOD'] in self.conf['methods']:
|
655
717
|
try:
|
656
|
-
ver, acc, cont, obj = split_path(
|
718
|
+
ver, acc, cont, obj = split_path(
|
719
|
+
env['PATH_INFO'], 3 if allow_container_root else 4,
|
720
|
+
4, True)
|
657
721
|
except ValueError:
|
658
722
|
return (None, None, None)
|
659
|
-
if ver == 'v1' and obj.strip('/'):
|
660
|
-
return (wsgi_to_str(acc), wsgi_to_str(cont),
|
723
|
+
if ver == 'v1' and (allow_container_root or obj.strip('/')):
|
724
|
+
return (wsgi_to_str(acc), wsgi_to_str(cont),
|
725
|
+
wsgi_to_str(obj) if obj else '')
|
661
726
|
return (None, None, None)
|
662
727
|
|
663
|
-
def _get_temp_url_info(self, env):
|
664
|
-
"""
|
665
|
-
Returns the provided temporary URL parameters (sig, expires, prefix,
|
666
|
-
temp_url_ip_range), if given and syntactically valid.
|
667
|
-
Either sig, expires or prefix could be None if not provided.
|
668
|
-
If provided, expires is also converted to an int if possible or 0
|
669
|
-
if not, and checked for expiration (returns 0 if expired).
|
670
|
-
|
671
|
-
:param env: The WSGI environment for the request.
|
672
|
-
:returns: (sig, expires, prefix, filename, inline,
|
673
|
-
temp_url_ip_range) as described above.
|
674
|
-
"""
|
675
|
-
temp_url_sig = temp_url_expires = temp_url_prefix = filename =\
|
676
|
-
inline = None
|
677
|
-
temp_url_ip_range = None
|
678
|
-
qs = parse_qs(env.get('QUERY_STRING', ''), keep_blank_values=True)
|
679
|
-
if 'temp_url_ip_range' in qs:
|
680
|
-
temp_url_ip_range = qs['temp_url_ip_range'][0]
|
681
|
-
if 'temp_url_sig' in qs:
|
682
|
-
temp_url_sig = qs['temp_url_sig'][0]
|
683
|
-
if 'temp_url_expires' in qs:
|
684
|
-
try:
|
685
|
-
temp_url_expires = int(qs['temp_url_expires'][0])
|
686
|
-
except ValueError:
|
687
|
-
try:
|
688
|
-
temp_url_expires = timegm(strptime(
|
689
|
-
qs['temp_url_expires'][0],
|
690
|
-
EXPIRES_ISO8601_FORMAT))
|
691
|
-
except ValueError:
|
692
|
-
temp_url_expires = 0
|
693
|
-
if temp_url_expires < time():
|
694
|
-
temp_url_expires = 0
|
695
|
-
if 'temp_url_prefix' in qs:
|
696
|
-
temp_url_prefix = qs['temp_url_prefix'][0]
|
697
|
-
if 'filename' in qs:
|
698
|
-
filename = qs['filename'][0]
|
699
|
-
if 'inline' in qs:
|
700
|
-
inline = True
|
701
|
-
return (temp_url_sig, temp_url_expires, temp_url_prefix, filename,
|
702
|
-
inline, temp_url_ip_range)
|
703
|
-
|
704
728
|
def _get_keys(self, env):
|
705
729
|
"""
|
706
730
|
Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
|
@@ -750,12 +774,10 @@ class TempURL(object):
|
|
750
774
|
if not request_method:
|
751
775
|
request_method = env['REQUEST_METHOD']
|
752
776
|
|
753
|
-
digest = functools.partial(hashlib.new, hash_algorithm)
|
754
|
-
|
755
777
|
return [
|
756
778
|
(get_hmac(
|
757
779
|
request_method, path, expires, key,
|
758
|
-
digest=
|
780
|
+
digest=hash_algorithm, ip_range=ip_range
|
759
781
|
), scope)
|
760
782
|
for (key, scope) in scoped_keys]
|
761
783
|
|
@@ -839,7 +861,7 @@ class TempURL(object):
|
|
839
861
|
if h.startswith(p):
|
840
862
|
del headers[h]
|
841
863
|
break
|
842
|
-
return headers.items()
|
864
|
+
return list(headers.items())
|
843
865
|
|
844
866
|
|
845
867
|
def filter_factory(global_conf, **local_conf):
|
@@ -847,30 +869,26 @@ def filter_factory(global_conf, **local_conf):
|
|
847
869
|
conf = global_conf.copy()
|
848
870
|
conf.update(local_conf)
|
849
871
|
|
872
|
+
logger = get_logger(conf, log_route='tempurl')
|
873
|
+
|
850
874
|
defaults = {
|
851
875
|
'methods': 'GET HEAD PUT POST DELETE',
|
852
876
|
'incoming_remove_headers': DEFAULT_INCOMING_REMOVE_HEADERS,
|
853
877
|
'incoming_allow_headers': DEFAULT_INCOMING_ALLOW_HEADERS,
|
854
878
|
'outgoing_remove_headers': DEFAULT_OUTGOING_REMOVE_HEADERS,
|
855
879
|
'outgoing_allow_headers': DEFAULT_OUTGOING_ALLOW_HEADERS,
|
856
|
-
'allowed_digests': DEFAULT_ALLOWED_DIGESTS,
|
857
880
|
}
|
858
881
|
info_conf = {k: conf.get(k, v).split() for k, v in defaults.items()}
|
859
882
|
|
860
|
-
allowed_digests =
|
861
|
-
|
862
|
-
not_supported = allowed_digests - SUPPORTED_DIGESTS
|
863
|
-
if not_supported:
|
864
|
-
logger = get_logger(conf, log_route='tempurl')
|
865
|
-
logger.warning('The following digest algorithms are configured but '
|
866
|
-
'not supported: %s', ', '.join(not_supported))
|
867
|
-
allowed_digests -= not_supported
|
868
|
-
if not allowed_digests:
|
869
|
-
raise ValueError('No valid digest algorithms are configured '
|
870
|
-
'for tempurls')
|
883
|
+
allowed_digests, deprecated_digests = get_allowed_digests(
|
884
|
+
conf.get('allowed_digests', '').split(), logger)
|
871
885
|
info_conf['allowed_digests'] = sorted(allowed_digests)
|
886
|
+
if deprecated_digests:
|
887
|
+
info_conf['deprecated_digests'] = sorted(deprecated_digests)
|
872
888
|
|
873
889
|
register_swift_info('tempurl', **info_conf)
|
874
890
|
conf.update(info_conf)
|
875
891
|
|
876
|
-
|
892
|
+
register_sensitive_param('temp_url_sig')
|
893
|
+
|
894
|
+
return lambda app: TempURL(app, conf, logger)
|