swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/__init__.py +29 -50
- swift/account/auditor.py +21 -118
- swift/account/backend.py +33 -28
- swift/account/reaper.py +37 -28
- swift/account/replicator.py +22 -0
- swift/account/server.py +60 -26
- swift/account/utils.py +28 -11
- swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
- swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
- swift/cli/container_deleter.py +5 -11
- swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
- swift/cli/dispersion_report.py +10 -9
- swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
- swift/cli/form_signature.py +3 -7
- swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
- swift/cli/info.py +183 -29
- swift/cli/manage_shard_ranges.py +708 -37
- swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
- swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
- swift/cli/recon.py +196 -67
- swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
- swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +807 -126
- swift/cli/reload.py +135 -0
- swift/cli/ringbuilder.py +217 -20
- swift/cli/ringcomposer.py +0 -1
- swift/cli/shard-info.py +4 -3
- swift/common/base_storage_server.py +9 -20
- swift/common/bufferedhttp.py +48 -74
- swift/common/constraints.py +20 -15
- swift/common/container_sync_realms.py +9 -11
- swift/common/daemon.py +25 -8
- swift/common/db.py +198 -127
- swift/common/db_auditor.py +168 -0
- swift/common/db_replicator.py +95 -55
- swift/common/digest.py +141 -0
- swift/common/direct_client.py +144 -33
- swift/common/error_limiter.py +93 -0
- swift/common/exceptions.py +25 -1
- swift/common/header_key_dict.py +2 -9
- swift/common/http_protocol.py +373 -0
- swift/common/internal_client.py +129 -59
- swift/common/linkat.py +3 -4
- swift/common/manager.py +284 -67
- swift/common/memcached.py +396 -147
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +211 -46
- swift/common/middleware/acl.py +3 -8
- swift/common/middleware/backend_ratelimit.py +230 -0
- swift/common/middleware/bulk.py +22 -34
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +6 -11
- swift/common/middleware/container_quotas.py +1 -1
- swift/common/middleware/container_sync.py +39 -17
- swift/common/middleware/copy.py +12 -0
- swift/common/middleware/crossdomain.py +22 -9
- swift/common/middleware/crypto/__init__.py +2 -1
- swift/common/middleware/crypto/crypto_utils.py +11 -15
- swift/common/middleware/crypto/decrypter.py +28 -11
- swift/common/middleware/crypto/encrypter.py +12 -17
- swift/common/middleware/crypto/keymaster.py +8 -15
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/dlo.py +15 -11
- swift/common/middleware/domain_remap.py +5 -4
- swift/common/middleware/etag_quoter.py +128 -0
- swift/common/middleware/formpost.py +73 -70
- swift/common/middleware/gatekeeper.py +8 -1
- swift/common/middleware/keystoneauth.py +33 -3
- swift/common/middleware/list_endpoints.py +4 -4
- swift/common/middleware/listing_formats.py +85 -49
- swift/common/middleware/memcache.py +4 -81
- swift/common/middleware/name_check.py +3 -2
- swift/common/middleware/proxy_logging.py +160 -92
- swift/common/middleware/ratelimit.py +17 -10
- swift/common/middleware/read_only.py +6 -4
- swift/common/middleware/recon.py +59 -22
- swift/common/middleware/s3api/acl_handlers.py +25 -3
- swift/common/middleware/s3api/acl_utils.py +6 -1
- swift/common/middleware/s3api/controllers/__init__.py +6 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/bucket.py +242 -137
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
- swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
- swift/common/middleware/s3api/controllers/obj.py +112 -8
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
- swift/common/middleware/s3api/controllers/tagging.py +57 -0
- swift/common/middleware/s3api/controllers/versioning.py +36 -7
- swift/common/middleware/s3api/etree.py +22 -9
- swift/common/middleware/s3api/exception.py +0 -4
- swift/common/middleware/s3api/s3api.py +113 -41
- swift/common/middleware/s3api/s3request.py +384 -218
- swift/common/middleware/s3api/s3response.py +126 -23
- swift/common/middleware/s3api/s3token.py +16 -17
- swift/common/middleware/s3api/schema/delete.rng +1 -1
- swift/common/middleware/s3api/subresource.py +7 -10
- swift/common/middleware/s3api/utils.py +27 -10
- swift/common/middleware/slo.py +665 -358
- swift/common/middleware/staticweb.py +64 -37
- swift/common/middleware/symlink.py +52 -19
- swift/common/middleware/tempauth.py +76 -58
- swift/common/middleware/tempurl.py +192 -174
- swift/common/middleware/versioned_writes/__init__.py +51 -0
- swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
- swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +18 -19
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +10 -10
- swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
- swift/common/registry.py +147 -0
- swift/common/request_helpers.py +324 -57
- swift/common/ring/builder.py +67 -25
- swift/common/ring/composite_builder.py +1 -1
- swift/common/ring/ring.py +177 -51
- swift/common/ring/utils.py +1 -1
- swift/common/splice.py +10 -6
- swift/common/statsd_client.py +205 -0
- swift/common/storage_policy.py +49 -44
- swift/common/swob.py +86 -102
- swift/common/{utils.py → utils/__init__.py} +2191 -2762
- swift/common/utils/base.py +131 -0
- swift/common/utils/config.py +433 -0
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/logs.py +859 -0
- swift/common/utils/timestamp.py +412 -0
- swift/common/wsgi.py +555 -536
- swift/container/auditor.py +14 -100
- swift/container/backend.py +552 -227
- swift/container/reconciler.py +126 -37
- swift/container/replicator.py +96 -22
- swift/container/server.py +397 -176
- swift/container/sharder.py +1580 -639
- swift/container/sync.py +94 -88
- swift/container/updater.py +53 -32
- swift/obj/auditor.py +153 -35
- swift/obj/diskfile.py +466 -217
- swift/obj/expirer.py +406 -124
- swift/obj/mem_diskfile.py +7 -4
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +523 -262
- swift/obj/replicator.py +249 -188
- swift/obj/server.py +213 -122
- swift/obj/ssync_receiver.py +145 -85
- swift/obj/ssync_sender.py +113 -54
- swift/obj/updater.py +653 -139
- swift/obj/watchers/__init__.py +0 -0
- swift/obj/watchers/dark_data.py +213 -0
- swift/proxy/controllers/account.py +11 -11
- swift/proxy/controllers/base.py +848 -604
- swift/proxy/controllers/container.py +452 -86
- swift/proxy/controllers/info.py +3 -2
- swift/proxy/controllers/obj.py +1009 -490
- swift/proxy/server.py +185 -112
- swift-2.35.0.dist-info/AUTHORS +501 -0
- swift-2.35.0.dist-info/LICENSE +202 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
- swift-2.35.0.dist-info/RECORD +201 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
- swift-2.35.0.dist-info/pbr.json +1 -0
- swift/locale/de/LC_MESSAGES/swift.po +0 -1216
- swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
- swift/locale/es/LC_MESSAGES/swift.po +0 -1085
- swift/locale/fr/LC_MESSAGES/swift.po +0 -909
- swift/locale/it/LC_MESSAGES/swift.po +0 -894
- swift/locale/ja/LC_MESSAGES/swift.po +0 -965
- swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
- swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
- swift/locale/ru/LC_MESSAGES/swift.po +0 -891
- swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
- swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
- swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
- swift-2.23.2.data/scripts/swift-account-auditor +0 -23
- swift-2.23.2.data/scripts/swift-account-info +0 -51
- swift-2.23.2.data/scripts/swift-account-reaper +0 -23
- swift-2.23.2.data/scripts/swift-account-replicator +0 -34
- swift-2.23.2.data/scripts/swift-account-server +0 -23
- swift-2.23.2.data/scripts/swift-container-auditor +0 -23
- swift-2.23.2.data/scripts/swift-container-info +0 -51
- swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
- swift-2.23.2.data/scripts/swift-container-replicator +0 -34
- swift-2.23.2.data/scripts/swift-container-sharder +0 -33
- swift-2.23.2.data/scripts/swift-container-sync +0 -23
- swift-2.23.2.data/scripts/swift-container-updater +0 -23
- swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
- swift-2.23.2.data/scripts/swift-form-signature +0 -20
- swift-2.23.2.data/scripts/swift-init +0 -119
- swift-2.23.2.data/scripts/swift-object-auditor +0 -29
- swift-2.23.2.data/scripts/swift-object-expirer +0 -33
- swift-2.23.2.data/scripts/swift-object-info +0 -60
- swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
- swift-2.23.2.data/scripts/swift-object-relinker +0 -41
- swift-2.23.2.data/scripts/swift-object-replicator +0 -37
- swift-2.23.2.data/scripts/swift-object-server +0 -27
- swift-2.23.2.data/scripts/swift-object-updater +0 -23
- swift-2.23.2.data/scripts/swift-proxy-server +0 -23
- swift-2.23.2.data/scripts/swift-recon +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.23.2.data/scripts/swift-ring-composer +0 -22
- swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
- swift-2.23.2.dist-info/RECORD +0 -220
- swift-2.23.2.dist-info/metadata.json +0 -1
- swift-2.23.2.dist-info/pbr.json +0 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -25,9 +25,9 @@ from swift.common.request_helpers import get_object_transient_sysmeta, \
|
|
25
25
|
strip_user_meta_prefix, is_user_meta, update_etag_is_at_header, \
|
26
26
|
get_container_update_override_key
|
27
27
|
from swift.common.swob import Request, Match, HTTPException, \
|
28
|
-
HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi
|
28
|
+
HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi, normalize_etag
|
29
29
|
from swift.common.utils import get_logger, config_true_value, \
|
30
|
-
MD5_OF_EMPTY_STRING
|
30
|
+
MD5_OF_EMPTY_STRING, md5, InputProxy
|
31
31
|
|
32
32
|
|
33
33
|
def encrypt_header_val(crypto, value, key):
|
@@ -66,11 +66,11 @@ def _hmac_etag(key, etag):
|
|
66
66
|
return base64.b64encode(result).decode()
|
67
67
|
|
68
68
|
|
69
|
-
class EncInputWrapper(
|
69
|
+
class EncInputWrapper(InputProxy):
|
70
70
|
"""File-like object to be swapped in for wsgi.input."""
|
71
71
|
def __init__(self, crypto, keys, req, logger):
|
72
|
+
super().__init__(req.environ['wsgi.input'])
|
72
73
|
self.env = req.environ
|
73
|
-
self.wsgi_input = req.environ['wsgi.input']
|
74
74
|
self.path = req.path
|
75
75
|
self.crypto = crypto
|
76
76
|
self.body_crypto_ctxt = None
|
@@ -91,8 +91,8 @@ class EncInputWrapper(object):
|
|
91
91
|
self.body_crypto_meta['key_id'] = self.keys['id']
|
92
92
|
self.body_crypto_ctxt = self.crypto.create_encryption_ctxt(
|
93
93
|
body_key, self.body_crypto_meta.get('iv'))
|
94
|
-
self.plaintext_md5 =
|
95
|
-
self.ciphertext_md5 =
|
94
|
+
self.plaintext_md5 = md5(usedforsecurity=False)
|
95
|
+
self.ciphertext_md5 = md5(usedforsecurity=False)
|
96
96
|
|
97
97
|
def install_footers_callback(self, req):
|
98
98
|
# the proxy controller will call back for footer metadata after
|
@@ -166,7 +166,7 @@ class EncInputWrapper(object):
|
|
166
166
|
# value, with the crypto parameters appended. We use the
|
167
167
|
# container key here so that only that key is required to
|
168
168
|
# decrypt all etag values in a container listing when handling
|
169
|
-
# a container GET request. Don't encrypt an
|
169
|
+
# a container GET request. Don't encrypt an MD5_OF_EMPTY_STRING
|
170
170
|
# unless there actually was some body content, in which case
|
171
171
|
# the container-listing etag is possibly conveying some
|
172
172
|
# non-obvious information.
|
@@ -180,15 +180,7 @@ class EncInputWrapper(object):
|
|
180
180
|
|
181
181
|
req.environ['swift.callback.update_footers'] = footers_callback
|
182
182
|
|
183
|
-
def
|
184
|
-
return self.readChunk(self.wsgi_input.read, *args, **kwargs)
|
185
|
-
|
186
|
-
def readline(self, *args, **kwargs):
|
187
|
-
return self.readChunk(self.wsgi_input.readline, *args, **kwargs)
|
188
|
-
|
189
|
-
def readChunk(self, read_method, *args, **kwargs):
|
190
|
-
chunk = read_method(*args, **kwargs)
|
191
|
-
|
183
|
+
def chunk_update(self, chunk, eof, *args, **kwargs):
|
192
184
|
if chunk:
|
193
185
|
self._init_encryption_context()
|
194
186
|
self.plaintext_md5.update(chunk)
|
@@ -263,7 +255,7 @@ class EncrypterObjContext(CryptoWSGIContext):
|
|
263
255
|
ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest()
|
264
256
|
mod_resp_headers = [
|
265
257
|
(h, v if (h.lower() != 'etag' or
|
266
|
-
v
|
258
|
+
normalize_etag(v) != ciphertext_etag)
|
267
259
|
else plaintext_etag)
|
268
260
|
for h, v in mod_resp_headers]
|
269
261
|
|
@@ -369,7 +361,10 @@ class Encrypter(object):
|
|
369
361
|
return self.app(env, start_response)
|
370
362
|
try:
|
371
363
|
req.split_path(4, 4, True)
|
364
|
+
is_object_request = True
|
372
365
|
except ValueError:
|
366
|
+
is_object_request = False
|
367
|
+
if not is_object_request:
|
373
368
|
return self.app(env, start_response)
|
374
369
|
|
375
370
|
if req.method in ('GET', 'HEAD'):
|
@@ -14,7 +14,6 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
import hashlib
|
16
16
|
import hmac
|
17
|
-
import six
|
18
17
|
|
19
18
|
from swift.common.exceptions import UnknownSecretIdError
|
20
19
|
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
|
@@ -101,14 +100,9 @@ class KeyMasterContext(WSGIContext):
|
|
101
100
|
# Older py3 proxies may have written down crypto meta as WSGI
|
102
101
|
# strings; we still need to be able to read that
|
103
102
|
try:
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
for part in (key_acct, key_cont, key_obj))
|
108
|
-
else:
|
109
|
-
alt_path = tuple(
|
110
|
-
part.encode('latin1').decode('utf-8')
|
111
|
-
for part in (key_acct, key_cont, key_obj))
|
103
|
+
alt_path = tuple(
|
104
|
+
part.encode('latin1').decode('utf-8')
|
105
|
+
for part in (key_acct, key_cont, key_obj))
|
112
106
|
except UnicodeError:
|
113
107
|
# Well, it was worth a shot
|
114
108
|
pass
|
@@ -208,10 +202,10 @@ class BaseKeyMaster(object):
|
|
208
202
|
|
209
203
|
This provides some basic helpers for:
|
210
204
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
205
|
+
- loading from a separate config path,
|
206
|
+
- deriving keys based on path, and
|
207
|
+
- installing a ``swift.callback.fetch_crypto_keys`` hook
|
208
|
+
in the request environment.
|
215
209
|
|
216
210
|
Subclasses should define ``log_route``, ``keymaster_opts``, and
|
217
211
|
``keymaster_conf_section`` attributes, and implement the
|
@@ -336,8 +330,7 @@ class BaseKeyMaster(object):
|
|
336
330
|
self.logger.warning('Unrecognised secret id: %s' % secret_id)
|
337
331
|
raise UnknownSecretIdError(secret_id)
|
338
332
|
else:
|
339
|
-
|
340
|
-
path = path.encode('utf-8')
|
333
|
+
path = path.encode('utf-8')
|
341
334
|
return hmac.new(key, path, digestmod=hashlib.sha256).digest()
|
342
335
|
|
343
336
|
|
@@ -34,7 +34,7 @@ class KmsKeyMaster(BaseKeyMaster):
|
|
34
34
|
'domain_id', 'domain_name', 'project_id',
|
35
35
|
'project_domain_id', 'reauthenticate',
|
36
36
|
'auth_endpoint', 'api_class', 'key_id*',
|
37
|
-
'active_root_secret_id')
|
37
|
+
'barbican_endpoint', 'active_root_secret_id')
|
38
38
|
keymaster_conf_section = 'kms_keymaster'
|
39
39
|
|
40
40
|
def _get_root_secret(self, conf):
|
@@ -67,6 +67,7 @@ class KmsKeyMaster(BaseKeyMaster):
|
|
67
67
|
oslo_conf = cfg.ConfigOpts()
|
68
68
|
options.set_defaults(
|
69
69
|
oslo_conf, auth_endpoint=conf.get('auth_endpoint'),
|
70
|
+
barbican_endpoint=conf.get('barbican_endpoint'),
|
70
71
|
api_class=conf.get('api_class')
|
71
72
|
)
|
72
73
|
options.enable_logging()
|
swift/common/middleware/dlo.py
CHANGED
@@ -120,18 +120,17 @@ Here's an example using ``curl`` with tiny 1-byte segments::
|
|
120
120
|
|
121
121
|
import json
|
122
122
|
|
123
|
-
import six
|
124
|
-
|
125
|
-
from hashlib import md5
|
126
123
|
from swift.common import constraints
|
127
124
|
from swift.common.exceptions import ListingIterError, SegmentError
|
128
125
|
from swift.common.http import is_success
|
129
|
-
from swift.common.swob import Request, Response, \
|
126
|
+
from swift.common.swob import Request, Response, HTTPException, \
|
130
127
|
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
|
131
|
-
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
|
128
|
+
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote, normalize_etag
|
132
129
|
from swift.common.utils import get_logger, \
|
133
|
-
RateLimitedIterator, quote, close_if_possible, closing_if_possible
|
134
|
-
|
130
|
+
RateLimitedIterator, quote, close_if_possible, closing_if_possible, \
|
131
|
+
drain_and_close, md5
|
132
|
+
from swift.common.request_helpers import SegmentedIterable, \
|
133
|
+
update_ignore_range_header
|
135
134
|
from swift.common.wsgi import WSGIContext, make_subrequest, load_app_config
|
136
135
|
|
137
136
|
|
@@ -206,8 +205,6 @@ class GetContext(WSGIContext):
|
|
206
205
|
break
|
207
206
|
|
208
207
|
seg_name = segment['name']
|
209
|
-
if six.PY2:
|
210
|
-
seg_name = seg_name.encode("utf-8")
|
211
208
|
|
212
209
|
# We deliberately omit the etag and size here;
|
213
210
|
# SegmentedIterable will check size and etag if
|
@@ -331,9 +328,9 @@ class GetContext(WSGIContext):
|
|
331
328
|
if have_complete_listing:
|
332
329
|
response_headers = [(h, v) for h, v in response_headers
|
333
330
|
if h.lower() != "etag"]
|
334
|
-
etag = md5()
|
331
|
+
etag = md5(usedforsecurity=False)
|
335
332
|
for seg_dict in segments:
|
336
|
-
etag.update(seg_dict['hash']
|
333
|
+
etag.update(normalize_etag(seg_dict['hash']).encode('utf8'))
|
337
334
|
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
|
338
335
|
|
339
336
|
app_iter = None
|
@@ -353,6 +350,8 @@ class GetContext(WSGIContext):
|
|
353
350
|
|
354
351
|
try:
|
355
352
|
app_iter.validate_first_segment()
|
353
|
+
except HTTPException as err_resp:
|
354
|
+
return err_resp
|
356
355
|
except (SegmentError, ListingIterError):
|
357
356
|
return HTTPConflict(request=req)
|
358
357
|
|
@@ -369,11 +368,16 @@ class GetContext(WSGIContext):
|
|
369
368
|
|
370
369
|
Otherwise, simply pass it through.
|
371
370
|
"""
|
371
|
+
update_ignore_range_header(req, 'X-Object-Manifest')
|
372
372
|
resp_iter = self._app_call(req.environ)
|
373
373
|
|
374
374
|
# make sure this response is for a dynamic large object manifest
|
375
375
|
for header, value in self._response_headers:
|
376
376
|
if (header.lower() == 'x-object-manifest'):
|
377
|
+
content_length = self._response_header_value('content-length')
|
378
|
+
if content_length is not None and int(content_length) < 1024:
|
379
|
+
# Go ahead and consume small bodies
|
380
|
+
drain_and_close(resp_iter)
|
377
381
|
close_if_possible(resp_iter)
|
378
382
|
response = self.get_or_head_response(
|
379
383
|
req, wsgi_to_str(wsgi_unquote(value)))
|
@@ -99,9 +99,9 @@ storage end points as sync destinations.
|
|
99
99
|
"""
|
100
100
|
|
101
101
|
from swift.common.middleware import RewriteContext
|
102
|
-
from swift.common.swob import Request, HTTPBadRequest
|
103
|
-
from swift.common.utils import config_true_value, list_from_csv
|
104
|
-
|
102
|
+
from swift.common.swob import Request, HTTPBadRequest, wsgi_quote
|
103
|
+
from swift.common.utils import config_true_value, list_from_csv
|
104
|
+
from swift.common.registry import register_swift_info
|
105
105
|
|
106
106
|
|
107
107
|
class _DomainRemapContext(RewriteContext):
|
@@ -192,7 +192,8 @@ class DomainRemapMiddleware(object):
|
|
192
192
|
new_path = '/'.join(new_path_parts)
|
193
193
|
env['PATH_INFO'] = new_path
|
194
194
|
|
195
|
-
context = _DomainRemapContext(
|
195
|
+
context = _DomainRemapContext(
|
196
|
+
self.app, wsgi_quote(requested_path), wsgi_quote(new_path))
|
196
197
|
return context.handle_request(env, start_response)
|
197
198
|
|
198
199
|
return self.app(env, start_response)
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# Copyright (c) 2010-2020 OpenStack Foundation
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
12
|
+
# implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
"""
|
17
|
+
This middleware fix the Etag header of responses so that it is RFC compliant.
|
18
|
+
`RFC 7232 <https://tools.ietf.org/html/rfc7232#section-2.3>`__ specifies that
|
19
|
+
the value of the Etag header must be double quoted.
|
20
|
+
|
21
|
+
It must be placed at the beggining of the pipeline, right after cache::
|
22
|
+
|
23
|
+
[pipeline:main]
|
24
|
+
pipeline = ... cache etag-quoter ...
|
25
|
+
|
26
|
+
[filter:etag-quoter]
|
27
|
+
use = egg:swift#etag_quoter
|
28
|
+
|
29
|
+
Set ``X-Account-Rfc-Compliant-Etags: true`` at the account
|
30
|
+
level to have any Etags in object responses be double quoted, as in
|
31
|
+
``"d41d8cd98f00b204e9800998ecf8427e"``. Alternatively, you may
|
32
|
+
only fix Etags in a single container by setting
|
33
|
+
``X-Container-Rfc-Compliant-Etags: true`` on the container.
|
34
|
+
This may be necessary for Swift to work properly with some CDNs.
|
35
|
+
|
36
|
+
Either option may also be explicitly *disabled*, so you may enable quoted
|
37
|
+
Etags account-wide as above but turn them off for individual containers
|
38
|
+
with ``X-Container-Rfc-Compliant-Etags: false``. This may be
|
39
|
+
useful if some subset of applications expect Etags to be bare MD5s.
|
40
|
+
"""
|
41
|
+
|
42
|
+
from swift.common.constraints import valid_api_version
|
43
|
+
from swift.common.http import is_success
|
44
|
+
from swift.common.swob import Request
|
45
|
+
from swift.common.utils import config_true_value
|
46
|
+
from swift.common.registry import register_swift_info
|
47
|
+
from swift.proxy.controllers.base import get_account_info, get_container_info
|
48
|
+
|
49
|
+
|
50
|
+
class EtagQuoterMiddleware(object):
|
51
|
+
def __init__(self, app, conf):
|
52
|
+
self.app = app
|
53
|
+
self.conf = conf
|
54
|
+
|
55
|
+
def __call__(self, env, start_response):
|
56
|
+
req = Request(env)
|
57
|
+
try:
|
58
|
+
version, account, container, obj = req.split_path(
|
59
|
+
2, 4, rest_with_last=True)
|
60
|
+
is_swifty_request = valid_api_version(version)
|
61
|
+
except ValueError:
|
62
|
+
is_swifty_request = False
|
63
|
+
|
64
|
+
if not is_swifty_request:
|
65
|
+
return self.app(env, start_response)
|
66
|
+
|
67
|
+
if not obj:
|
68
|
+
typ = 'Container' if container else 'Account'
|
69
|
+
client_header = 'X-%s-Rfc-Compliant-Etags' % typ
|
70
|
+
sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ
|
71
|
+
if client_header in req.headers:
|
72
|
+
if req.headers[client_header]:
|
73
|
+
req.headers[sysmeta_header] = config_true_value(
|
74
|
+
req.headers[client_header])
|
75
|
+
else:
|
76
|
+
req.headers[sysmeta_header] = ''
|
77
|
+
if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)):
|
78
|
+
req.headers[sysmeta_header] = ''
|
79
|
+
|
80
|
+
def translating_start_response(status, headers, exc_info=None):
|
81
|
+
return start_response(status, [
|
82
|
+
(client_header if h.title() == sysmeta_header else h,
|
83
|
+
v) for h, v in headers
|
84
|
+
], exc_info)
|
85
|
+
|
86
|
+
return self.app(env, translating_start_response)
|
87
|
+
|
88
|
+
container_info = get_container_info(env, self.app, 'EQ')
|
89
|
+
if not container_info or not is_success(container_info['status']):
|
90
|
+
return self.app(env, start_response)
|
91
|
+
|
92
|
+
flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags')
|
93
|
+
if flag is None:
|
94
|
+
account_info = get_account_info(env, self.app, 'EQ')
|
95
|
+
if not account_info or not is_success(account_info['status']):
|
96
|
+
return self.app(env, start_response)
|
97
|
+
|
98
|
+
flag = account_info.get('sysmeta', {}).get(
|
99
|
+
'rfc-compliant-etags')
|
100
|
+
|
101
|
+
if flag is None:
|
102
|
+
flag = self.conf.get('enable_by_default', 'false')
|
103
|
+
|
104
|
+
if not config_true_value(flag):
|
105
|
+
return self.app(env, start_response)
|
106
|
+
|
107
|
+
status, headers, resp_iter = req.call_application(self.app)
|
108
|
+
|
109
|
+
headers = [
|
110
|
+
(header, value) if header.lower() != 'etag' or (
|
111
|
+
value.startswith(('"', 'W/"')) and value.endswith('"'))
|
112
|
+
else (header, '"%s"' % value)
|
113
|
+
for header, value in headers]
|
114
|
+
|
115
|
+
start_response(status, headers)
|
116
|
+
return resp_iter
|
117
|
+
|
118
|
+
|
119
|
+
def filter_factory(global_conf, **local_conf):
|
120
|
+
conf = global_conf.copy()
|
121
|
+
conf.update(local_conf)
|
122
|
+
register_swift_info(
|
123
|
+
'etag_quoter', enable_by_default=config_true_value(
|
124
|
+
conf.get('enable_by_default', 'false')))
|
125
|
+
|
126
|
+
def etag_quoter_filter(app):
|
127
|
+
return EtagQuoterMiddleware(app, conf)
|
128
|
+
return etag_quoter_filter
|
@@ -84,11 +84,11 @@ desired.
|
|
84
84
|
The expires attribute is the Unix timestamp before which the form
|
85
85
|
must be submitted before it is invalidated.
|
86
86
|
|
87
|
-
The signature attribute is the HMAC
|
87
|
+
The signature attribute is the HMAC signature of the form. Here is
|
88
88
|
sample code for computing the signature::
|
89
89
|
|
90
90
|
import hmac
|
91
|
-
from hashlib import
|
91
|
+
from hashlib import sha512
|
92
92
|
from time import time
|
93
93
|
path = '/v1/account/container/object_prefix'
|
94
94
|
redirect = 'https://srv.com/some-page' # set to '' if redirect not in form
|
@@ -98,7 +98,7 @@ sample code for computing the signature::
|
|
98
98
|
key = 'mykey'
|
99
99
|
hmac_body = '%s\n%s\n%s\n%s\n%s' % (path, redirect,
|
100
100
|
max_file_size, max_file_count, expires)
|
101
|
-
signature = hmac.new(key, hmac_body,
|
101
|
+
signature = hmac.new(key, hmac_body, sha512).hexdigest()
|
102
102
|
|
103
103
|
The key is the value of either the account (X-Account-Meta-Temp-URL-Key,
|
104
104
|
X-Account-Meta-Temp-Url-Key-2) or the container
|
@@ -123,20 +123,22 @@ the file are simply ignored).
|
|
123
123
|
__all__ = ['FormPost', 'filter_factory', 'READ_CHUNK_SIZE', 'MAX_VALUE_LENGTH']
|
124
124
|
|
125
125
|
import hmac
|
126
|
-
from hashlib import sha1
|
127
126
|
from time import time
|
128
127
|
|
129
|
-
import
|
130
|
-
from six.moves.urllib.parse import quote
|
128
|
+
from urllib.parse import quote
|
131
129
|
|
132
130
|
from swift.common.constraints import valid_api_version
|
133
131
|
from swift.common.exceptions import MimeInvalid
|
134
132
|
from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
|
135
|
-
from swift.common.
|
136
|
-
|
137
|
-
|
138
|
-
|
133
|
+
from swift.common.digest import get_allowed_digests, \
|
134
|
+
extract_digest_and_algorithm, DEFAULT_ALLOWED_DIGESTS
|
135
|
+
from swift.common.utils import streq_const_time, parse_content_disposition, \
|
136
|
+
parse_mime_headers, iter_multipart_mime_documents, reiterate, \
|
137
|
+
closing_if_possible, get_logger, InputProxy
|
138
|
+
from swift.common.registry import register_swift_info
|
139
|
+
from swift.common.wsgi import WSGIContext, make_pre_authed_env
|
139
140
|
from swift.common.swob import HTTPUnauthorized, wsgi_to_str, str_to_wsgi
|
141
|
+
from swift.common.http import is_success
|
140
142
|
from swift.proxy.controllers.base import get_account_info, get_container_info
|
141
143
|
|
142
144
|
|
@@ -156,7 +158,7 @@ class FormUnauthorized(Exception):
|
|
156
158
|
pass
|
157
159
|
|
158
160
|
|
159
|
-
class _CappedFileLikeObject(
|
161
|
+
class _CappedFileLikeObject(InputProxy):
|
160
162
|
"""
|
161
163
|
A file-like object wrapping another file-like object that raises
|
162
164
|
an EOFError if the amount of data read exceeds a given
|
@@ -168,26 +170,15 @@ class _CappedFileLikeObject(object):
|
|
168
170
|
"""
|
169
171
|
|
170
172
|
def __init__(self, fp, max_file_size):
|
171
|
-
|
173
|
+
super().__init__(fp)
|
172
174
|
self.max_file_size = max_file_size
|
173
|
-
self.amount_read = 0
|
174
175
|
self.file_size_exceeded = False
|
175
176
|
|
176
|
-
def
|
177
|
-
|
178
|
-
self.amount_read += len(ret)
|
179
|
-
if self.amount_read > self.max_file_size:
|
177
|
+
def chunk_update(self, chunk, eof, *args, **kwargs):
|
178
|
+
if self.bytes_received > self.max_file_size:
|
180
179
|
self.file_size_exceeded = True
|
181
180
|
raise EOFError('max_file_size exceeded')
|
182
|
-
return
|
183
|
-
|
184
|
-
def readline(self):
|
185
|
-
ret = self.fp.readline()
|
186
|
-
self.amount_read += len(ret)
|
187
|
-
if self.amount_read > self.max_file_size:
|
188
|
-
self.file_size_exceeded = True
|
189
|
-
raise EOFError('max_file_size exceeded')
|
190
|
-
return ret
|
181
|
+
return chunk
|
191
182
|
|
192
183
|
|
193
184
|
class FormPost(object):
|
@@ -204,11 +195,17 @@ class FormPost(object):
|
|
204
195
|
:param conf: The configuration dict for the middleware.
|
205
196
|
"""
|
206
197
|
|
207
|
-
def __init__(self, app, conf):
|
198
|
+
def __init__(self, app, conf, logger=None):
|
208
199
|
#: The next WSGI application/filter in the paste.deploy pipeline.
|
209
200
|
self.app = app
|
210
201
|
#: The filter configuration dict.
|
211
202
|
self.conf = conf
|
203
|
+
self.logger = logger or get_logger(conf, log_route='formpost')
|
204
|
+
# Defaulting to SUPPORTED_DIGESTS just so we don't completely
|
205
|
+
# deprecate sha1 yet. We'll change this to DEFAULT_ALLOWED_DIGESTS
|
206
|
+
# later.
|
207
|
+
self.allowed_digests = conf.get(
|
208
|
+
'allowed_digests', DEFAULT_ALLOWED_DIGESTS.split())
|
212
209
|
|
213
210
|
def __call__(self, env, start_response):
|
214
211
|
"""
|
@@ -239,9 +236,7 @@ class FormPost(object):
|
|
239
236
|
('Content-Length', str(len(body)))))
|
240
237
|
return [body]
|
241
238
|
except (FormInvalid, EOFError) as err:
|
242
|
-
body = 'FormPost: %s' % err
|
243
|
-
if six.PY3:
|
244
|
-
body = body.encode('utf-8')
|
239
|
+
body = ('FormPost: %s' % err).encode('utf-8')
|
245
240
|
start_response(
|
246
241
|
'400 Bad Request',
|
247
242
|
(('Content-Type', 'text/plain'),
|
@@ -263,11 +258,12 @@ class FormPost(object):
|
|
263
258
|
:returns: status_line, headers_list, body
|
264
259
|
"""
|
265
260
|
keys = self._get_keys(env)
|
266
|
-
|
267
|
-
boundary = boundary.encode('utf-8')
|
261
|
+
boundary = boundary.encode('utf-8')
|
268
262
|
status = message = ''
|
269
263
|
attributes = {}
|
264
|
+
file_attributes = {}
|
270
265
|
subheaders = []
|
266
|
+
resp_body = None
|
271
267
|
file_count = 0
|
272
268
|
for fp in iter_multipart_mime_documents(
|
273
269
|
env['wsgi.input'], boundary, read_chunk_size=READ_CHUNK_SIZE):
|
@@ -283,16 +279,19 @@ class FormPost(object):
|
|
283
279
|
break
|
284
280
|
except ValueError:
|
285
281
|
raise FormInvalid('max_file_count not an integer')
|
286
|
-
|
282
|
+
file_attributes = attributes.copy()
|
283
|
+
file_attributes['filename'] = attrs['filename'] or 'filename'
|
287
284
|
if 'content-type' not in attributes and 'content-type' in hdrs:
|
288
|
-
|
285
|
+
file_attributes['content-type'] = \
|
289
286
|
hdrs['Content-Type'] or 'application/octet-stream'
|
290
287
|
if 'content-encoding' not in attributes and \
|
291
288
|
'content-encoding' in hdrs:
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
289
|
+
file_attributes['content-encoding'] = \
|
290
|
+
hdrs['Content-Encoding']
|
291
|
+
status, subheaders, resp_body = \
|
292
|
+
self._perform_subrequest(env, file_attributes, fp, keys)
|
293
|
+
status_code = int(status.split(' ', 1)[0])
|
294
|
+
if not is_success(status_code):
|
296
295
|
break
|
297
296
|
else:
|
298
297
|
data = b''
|
@@ -305,14 +304,14 @@ class FormPost(object):
|
|
305
304
|
data += chunk
|
306
305
|
while fp.read(READ_CHUNK_SIZE):
|
307
306
|
pass
|
308
|
-
|
309
|
-
data = data.decode('utf-8')
|
307
|
+
data = data.decode('utf-8')
|
310
308
|
if 'name' in attrs:
|
311
309
|
attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
|
312
310
|
if not status:
|
313
311
|
status = '400 Bad Request'
|
314
312
|
message = 'no files to process'
|
315
313
|
|
314
|
+
status_code = int(status.split(' ', 1)[0])
|
316
315
|
headers = [(k, v) for k, v in subheaders
|
317
316
|
if k.lower().startswith('access-control')]
|
318
317
|
|
@@ -321,21 +320,21 @@ class FormPost(object):
|
|
321
320
|
body = status
|
322
321
|
if message:
|
323
322
|
body = status + '\r\nFormPost: ' + message.title()
|
323
|
+
body = body.encode('utf-8')
|
324
|
+
if not is_success(status_code) and resp_body:
|
325
|
+
body = resp_body
|
324
326
|
headers.extend([('Content-Type', 'text/plain'),
|
325
327
|
('Content-Length', len(body))])
|
326
|
-
if six.PY3:
|
327
|
-
body = body.encode('utf-8')
|
328
328
|
return status, headers, body
|
329
|
-
status = status.split(' ', 1)[0]
|
330
329
|
if '?' in redirect:
|
331
330
|
redirect += '&'
|
332
331
|
else:
|
333
332
|
redirect += '?'
|
334
|
-
redirect += 'status=%s&message=%s' % (quote(
|
333
|
+
redirect += 'status=%s&message=%s' % (quote(str(status_code)),
|
334
|
+
quote(message))
|
335
335
|
body = '<html><body><p><a href="%s">' \
|
336
336
|
'Click to continue...</a></p></body></html>' % redirect
|
337
|
-
|
338
|
-
body = body.encode('utf-8')
|
337
|
+
body = body.encode('utf-8')
|
339
338
|
headers.extend(
|
340
339
|
[('Location', redirect), ('Content-Length', str(len(body)))])
|
341
340
|
return '303 See Other', headers, body
|
@@ -397,37 +396,35 @@ class FormPost(object):
|
|
397
396
|
attributes.get('max_file_size') or '0',
|
398
397
|
attributes.get('max_file_count') or '0',
|
399
398
|
attributes.get('expires') or '0')
|
400
|
-
|
401
|
-
hmac_body = hmac_body.encode('utf-8')
|
399
|
+
hmac_body = hmac_body.encode('utf-8')
|
402
400
|
|
403
401
|
has_valid_sig = False
|
402
|
+
signature = attributes.get('signature', '')
|
403
|
+
try:
|
404
|
+
hash_name, signature = extract_digest_and_algorithm(signature)
|
405
|
+
except ValueError:
|
406
|
+
raise FormUnauthorized('invalid signature')
|
407
|
+
if hash_name not in self.allowed_digests:
|
408
|
+
raise FormUnauthorized('invalid signature')
|
409
|
+
|
404
410
|
for key in keys:
|
405
411
|
# Encode key like in swift.common.utls.get_hmac.
|
406
|
-
if not isinstance(key,
|
412
|
+
if not isinstance(key, bytes):
|
407
413
|
key = key.encode('utf8')
|
408
|
-
sig = hmac.new(key, hmac_body,
|
409
|
-
if streq_const_time(sig,
|
410
|
-
'invalid')):
|
414
|
+
sig = hmac.new(key, hmac_body, hash_name).hexdigest()
|
415
|
+
if streq_const_time(sig, signature):
|
411
416
|
has_valid_sig = True
|
412
417
|
if not has_valid_sig:
|
413
418
|
raise FormUnauthorized('invalid signature')
|
414
|
-
|
415
|
-
|
416
|
-
subheaders = [None]
|
417
|
-
|
419
|
+
self.logger.increment('formpost.digests.%s' % hash_name)
|
420
|
+
wsgi_ctx = WSGIContext(self.app)
|
418
421
|
wsgi_input = subenv['wsgi.input']
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
subheaders[0] = headers
|
426
|
-
|
427
|
-
# reiterate to ensure the response started,
|
428
|
-
# but drop any data on the floor
|
429
|
-
close_if_possible(reiterate(self.app(subenv, _start_response)))
|
430
|
-
return substatus[0], subheaders[0]
|
422
|
+
resp = wsgi_ctx._app_call(subenv)
|
423
|
+
if wsgi_input.file_size_exceeded:
|
424
|
+
raise EOFError("max_file_size exceeded")
|
425
|
+
with closing_if_possible(reiterate(resp)):
|
426
|
+
body = b''.join(resp)
|
427
|
+
return wsgi_ctx._response_status, wsgi_ctx._response_headers, body
|
431
428
|
|
432
429
|
def _get_keys(self, env):
|
433
430
|
"""
|
@@ -463,6 +460,12 @@ def filter_factory(global_conf, **local_conf):
|
|
463
460
|
conf = global_conf.copy()
|
464
461
|
conf.update(local_conf)
|
465
462
|
|
466
|
-
|
467
|
-
|
463
|
+
logger = get_logger(conf, log_route='formpost')
|
464
|
+
allowed_digests, deprecated_digests = get_allowed_digests(
|
465
|
+
conf.get('allowed_digests', '').split(), logger)
|
466
|
+
info = {'allowed_digests': sorted(allowed_digests)}
|
467
|
+
if deprecated_digests:
|
468
|
+
info['deprecated_digests'] = sorted(deprecated_digests)
|
469
|
+
register_swift_info('formpost', **info)
|
470
|
+
conf.update(info)
|
468
471
|
return lambda app: FormPost(app, conf)
|