swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/account/auditor.py +11 -0
- swift/account/reaper.py +11 -1
- swift/account/replicator.py +22 -0
- swift/account/server.py +13 -12
- swift-2.32.0.data/scripts/swift-account-audit → swift/cli/account_audit.py +6 -2
- swift-2.32.0.data/scripts/swift-config → swift/cli/config.py +1 -1
- swift-2.32.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +6 -2
- swift-2.32.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +12 -3
- swift-2.32.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +6 -2
- swift/cli/info.py +131 -3
- swift-2.32.0.data/scripts/swift-oldies → swift/cli/oldies.py +6 -3
- swift-2.32.0.data/scripts/swift-orphans → swift/cli/orphans.py +7 -2
- swift-2.32.0.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +9 -18
- swift-2.32.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +1 -1
- swift/cli/reload.py +141 -0
- swift/cli/ringbuilder.py +24 -0
- swift/common/daemon.py +12 -2
- swift/common/db.py +14 -9
- swift/common/db_auditor.py +2 -2
- swift/common/db_replicator.py +6 -0
- swift/common/exceptions.py +12 -0
- swift/common/http_protocol.py +76 -3
- swift/common/manager.py +120 -5
- swift/common/memcached.py +24 -25
- swift/common/middleware/account_quotas.py +144 -43
- swift/common/middleware/backend_ratelimit.py +166 -24
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +3 -5
- swift/common/middleware/container_sync.py +6 -10
- swift/common/middleware/crypto/crypto_utils.py +4 -5
- swift/common/middleware/crypto/decrypter.py +4 -5
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/proxy_logging.py +57 -43
- swift/common/middleware/ratelimit.py +6 -7
- swift/common/middleware/recon.py +6 -7
- swift/common/middleware/s3api/acl_handlers.py +10 -1
- swift/common/middleware/s3api/controllers/__init__.py +3 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_upload.py +31 -15
- swift/common/middleware/s3api/controllers/obj.py +20 -1
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/s3api.py +6 -0
- swift/common/middleware/s3api/s3request.py +190 -74
- swift/common/middleware/s3api/s3response.py +48 -8
- swift/common/middleware/s3api/s3token.py +2 -2
- swift/common/middleware/s3api/utils.py +2 -1
- swift/common/middleware/slo.py +508 -310
- swift/common/middleware/staticweb.py +45 -14
- swift/common/middleware/tempauth.py +6 -4
- swift/common/middleware/tempurl.py +134 -93
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +9 -10
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +1 -2
- swift/common/request_helpers.py +101 -8
- swift/common/statsd_client.py +207 -0
- swift/common/storage_policy.py +1 -1
- swift/common/swob.py +5 -2
- swift/common/utils/__init__.py +331 -1774
- swift/common/utils/base.py +138 -0
- swift/common/utils/config.py +443 -0
- swift/common/utils/logs.py +999 -0
- swift/common/utils/timestamp.py +23 -2
- swift/common/wsgi.py +19 -3
- swift/container/auditor.py +11 -0
- swift/container/backend.py +136 -31
- swift/container/reconciler.py +11 -2
- swift/container/replicator.py +64 -7
- swift/container/server.py +276 -146
- swift/container/sharder.py +86 -42
- swift/container/sync.py +11 -1
- swift/container/updater.py +12 -2
- swift/obj/auditor.py +20 -3
- swift/obj/diskfile.py +63 -25
- swift/obj/expirer.py +154 -47
- swift/obj/mem_diskfile.py +2 -1
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +28 -4
- swift/obj/replicator.py +63 -24
- swift/obj/server.py +76 -59
- swift/obj/updater.py +12 -2
- swift/obj/watchers/dark_data.py +72 -34
- swift/proxy/controllers/account.py +3 -2
- swift/proxy/controllers/base.py +254 -148
- swift/proxy/controllers/container.py +274 -289
- swift/proxy/controllers/obj.py +120 -166
- swift/proxy/server.py +17 -13
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/AUTHORS +14 -4
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/METADATA +9 -7
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/RECORD +97 -120
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/entry_points.txt +39 -0
- swift-2.34.0.dist-info/pbr.json +1 -0
- swift-2.32.0.data/scripts/swift-account-auditor +0 -23
- swift-2.32.0.data/scripts/swift-account-info +0 -52
- swift-2.32.0.data/scripts/swift-account-reaper +0 -23
- swift-2.32.0.data/scripts/swift-account-replicator +0 -34
- swift-2.32.0.data/scripts/swift-account-server +0 -23
- swift-2.32.0.data/scripts/swift-container-auditor +0 -23
- swift-2.32.0.data/scripts/swift-container-info +0 -56
- swift-2.32.0.data/scripts/swift-container-reconciler +0 -21
- swift-2.32.0.data/scripts/swift-container-replicator +0 -34
- swift-2.32.0.data/scripts/swift-container-server +0 -23
- swift-2.32.0.data/scripts/swift-container-sharder +0 -37
- swift-2.32.0.data/scripts/swift-container-sync +0 -23
- swift-2.32.0.data/scripts/swift-container-updater +0 -23
- swift-2.32.0.data/scripts/swift-dispersion-report +0 -24
- swift-2.32.0.data/scripts/swift-form-signature +0 -20
- swift-2.32.0.data/scripts/swift-init +0 -119
- swift-2.32.0.data/scripts/swift-object-auditor +0 -29
- swift-2.32.0.data/scripts/swift-object-expirer +0 -33
- swift-2.32.0.data/scripts/swift-object-info +0 -60
- swift-2.32.0.data/scripts/swift-object-reconstructor +0 -33
- swift-2.32.0.data/scripts/swift-object-relinker +0 -23
- swift-2.32.0.data/scripts/swift-object-replicator +0 -37
- swift-2.32.0.data/scripts/swift-object-server +0 -27
- swift-2.32.0.data/scripts/swift-object-updater +0 -23
- swift-2.32.0.data/scripts/swift-proxy-server +0 -23
- swift-2.32.0.data/scripts/swift-recon +0 -24
- swift-2.32.0.data/scripts/swift-ring-builder +0 -37
- swift-2.32.0.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.32.0.data/scripts/swift-ring-composer +0 -22
- swift-2.32.0.dist-info/pbr.json +0 -1
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/LICENSE +0 -0
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/WHEEL +0 -0
- {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/top_level.txt +0 -0
@@ -22,7 +22,6 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
|
22
22
|
import six
|
23
23
|
from six.moves.urllib import parse as urlparse
|
24
24
|
|
25
|
-
from swift import gettext_ as _
|
26
25
|
from swift.common.exceptions import EncryptionException, UnknownSecretIdError
|
27
26
|
from swift.common.swob import HTTPInternalServerError
|
28
27
|
from swift.common.utils import get_logger
|
@@ -160,7 +159,7 @@ class CryptoWSGIContext(WSGIContext):
|
|
160
159
|
try:
|
161
160
|
fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
|
162
161
|
except KeyError:
|
163
|
-
self.logger.exception(
|
162
|
+
self.logger.exception('ERROR get_keys() missing callback')
|
164
163
|
raise HTTPInternalServerError(
|
165
164
|
"Unable to retrieve encryption keys.")
|
166
165
|
|
@@ -181,12 +180,12 @@ class CryptoWSGIContext(WSGIContext):
|
|
181
180
|
self.crypto.check_key(key)
|
182
181
|
continue
|
183
182
|
except KeyError:
|
184
|
-
self.logger.exception(
|
183
|
+
self.logger.exception("Missing key for %r", name)
|
185
184
|
except TypeError:
|
186
|
-
self.logger.exception(
|
185
|
+
self.logger.exception("Did not get a keys dict")
|
187
186
|
except ValueError as e:
|
188
187
|
# don't include the key in any messages!
|
189
|
-
self.logger.exception(
|
188
|
+
self.logger.exception("Bad key for %(name)r: %(err)s",
|
190
189
|
{'name': name, 'err': e})
|
191
190
|
raise HTTPInternalServerError(
|
192
191
|
"Unable to retrieve encryption keys.")
|
@@ -16,7 +16,6 @@
|
|
16
16
|
import base64
|
17
17
|
import json
|
18
18
|
|
19
|
-
from swift import gettext_ as _
|
20
19
|
from swift.common.header_key_dict import HeaderKeyDict
|
21
20
|
from swift.common.http import is_success
|
22
21
|
from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
|
@@ -77,10 +76,10 @@ class BaseDecrypterContext(CryptoWSGIContext):
|
|
77
76
|
crypto_meta['body_key'])
|
78
77
|
except KeyError as err:
|
79
78
|
self.logger.error(
|
80
|
-
|
79
|
+
'Error decrypting %(resp_type)s: Missing %(key)s',
|
81
80
|
{'resp_type': self.server_type, 'key': err})
|
82
81
|
except ValueError as err:
|
83
|
-
self.logger.error(
|
82
|
+
self.logger.error('Error decrypting %(resp_type)s: %(reason)s',
|
84
83
|
{'resp_type': self.server_type, 'reason': err})
|
85
84
|
raise HTTPInternalServerError(
|
86
85
|
body='Error decrypting %s' % self.server_type,
|
@@ -178,7 +177,7 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
178
177
|
value, key, required, bytes_to_wsgi)
|
179
178
|
except EncryptionException as err:
|
180
179
|
self.logger.error(
|
181
|
-
|
180
|
+
"Error decrypting header %(header)s: %(error)s",
|
182
181
|
{'header': header, 'error': err})
|
183
182
|
raise HTTPInternalServerError(
|
184
183
|
body='Error decrypting header',
|
@@ -313,7 +312,7 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
313
312
|
try:
|
314
313
|
crypto_meta = self.get_crypto_meta(header, check)
|
315
314
|
except EncryptionException as err:
|
316
|
-
self.logger.error(
|
315
|
+
self.logger.error('Error decrypting object: %s', err)
|
317
316
|
raise HTTPInternalServerError(
|
318
317
|
body='Error decrypting object', content_type='text/plain')
|
319
318
|
return crypto_meta
|
@@ -34,7 +34,7 @@ class KmsKeyMaster(BaseKeyMaster):
|
|
34
34
|
'domain_id', 'domain_name', 'project_id',
|
35
35
|
'project_domain_id', 'reauthenticate',
|
36
36
|
'auth_endpoint', 'api_class', 'key_id*',
|
37
|
-
'active_root_secret_id')
|
37
|
+
'barbican_endpoint', 'active_root_secret_id')
|
38
38
|
keymaster_conf_section = 'kms_keymaster'
|
39
39
|
|
40
40
|
def _get_root_secret(self, conf):
|
@@ -67,6 +67,7 @@ class KmsKeyMaster(BaseKeyMaster):
|
|
67
67
|
oslo_conf = cfg.ConfigOpts()
|
68
68
|
options.set_defaults(
|
69
69
|
oslo_conf, auth_endpoint=conf.get('auth_endpoint'),
|
70
|
+
barbican_endpoint=conf.get('barbican_endpoint'),
|
70
71
|
api_class=conf.get('api_class')
|
71
72
|
)
|
72
73
|
options.enable_logging()
|
@@ -27,19 +27,28 @@ The logging format implemented below is as follows::
|
|
27
27
|
start_time end_time policy_index
|
28
28
|
|
29
29
|
These values are space-separated, and each is url-encoded, so that they can
|
30
|
-
be separated with a simple
|
31
|
-
|
32
|
-
* remote_addr is the contents of the REMOTE_ADDR environment variable,
|
33
|
-
client_ip is swift's best guess at the end-user IP, extracted
|
34
|
-
from the X-Forwarded-For header, X-Cluster-Ip header, or the
|
35
|
-
environment variable.
|
36
|
-
|
37
|
-
*
|
30
|
+
be separated with a simple ``.split()``.
|
31
|
+
|
32
|
+
* ``remote_addr`` is the contents of the REMOTE_ADDR environment variable,
|
33
|
+
while ``client_ip`` is swift's best guess at the end-user IP, extracted
|
34
|
+
variously from the X-Forwarded-For header, X-Cluster-Ip header, or the
|
35
|
+
REMOTE_ADDR environment variable.
|
36
|
+
|
37
|
+
* ``status_int`` is the integer part of the ``status`` string passed to this
|
38
|
+
middleware's start_response function, unless the WSGI environment has an item
|
39
|
+
with key ``swift.proxy_logging_status``, in which case the value of that item
|
40
|
+
is used. Other middleware's may set ``swift.proxy_logging_status`` to
|
41
|
+
override the logging of ``status_int``. In either case, the logged
|
42
|
+
``status_int`` value is forced to 499 if a client disconnect is detected
|
43
|
+
while this middleware is handling a request, or 500 if an exception is caught
|
44
|
+
while handling a request.
|
45
|
+
|
46
|
+
* ``source`` (``swift.source`` in the WSGI environment) indicates the code
|
38
47
|
that generated the request, such as most middleware. (See below for
|
39
48
|
more detail.)
|
40
49
|
|
41
|
-
* log_info (swift.log_info in the WSGI environment) is for additional
|
42
|
-
information that could prove quite useful, such as any x-delete-at
|
50
|
+
* ``log_info`` (``swift.log_info`` in the WSGI environment) is for additional
|
51
|
+
information that could prove quite useful, such as any ``x-delete-at``
|
43
52
|
value or other "behind the scenes" activity that might not
|
44
53
|
otherwise be detectable from the plain log information. Code that
|
45
54
|
wishes to add additional log information should use code like
|
@@ -62,24 +71,26 @@ For example, with staticweb, the middleware might intercept a request to
|
|
62
71
|
/v1/AUTH_acc/cont/, make a subrequest to the proxy to retrieve
|
63
72
|
/v1/AUTH_acc/cont/index.html and, in effect, respond to the client's original
|
64
73
|
request using the 2nd request's body. In this instance the subrequest will be
|
65
|
-
logged by the rightmost middleware (with a swift.source set) and the
|
66
|
-
request (with body overridden) will be logged by leftmost middleware.
|
74
|
+
logged by the rightmost middleware (with a ``swift.source`` set) and the
|
75
|
+
outgoing request (with body overridden) will be logged by leftmost middleware.
|
67
76
|
|
68
77
|
Requests that follow the normal pipeline (use the same wsgi environment
|
69
78
|
throughout) will not be double logged because an environment variable
|
70
|
-
(swift.proxy_access_log_made) is checked/set when a log is made.
|
79
|
+
(``swift.proxy_access_log_made``) is checked/set when a log is made.
|
71
80
|
|
72
|
-
All middleware making subrequests should take care to set swift.source when
|
81
|
+
All middleware making subrequests should take care to set ``swift.source`` when
|
73
82
|
needed. With the doubled proxy logs, any consumer/processor of swift's proxy
|
74
|
-
logs should look at the swift.source field, the rightmost log value, to
|
75
|
-
if this is a middleware subrequest or not. A log processor calculating
|
76
|
-
bandwidth usage will want to only sum up logs with no swift.source
|
83
|
+
logs should look at the ``swift.source`` field, the rightmost log value, to
|
84
|
+
decide if this is a middleware subrequest or not. A log processor calculating
|
85
|
+
bandwidth usage will want to only sum up logs with no ``swift.source``.
|
77
86
|
"""
|
78
87
|
|
79
88
|
import os
|
80
89
|
import time
|
81
90
|
|
91
|
+
from swift.common.constraints import valid_api_version
|
82
92
|
from swift.common.middleware.catch_errors import enforce_byte_count
|
93
|
+
from swift.common.request_helpers import get_log_info
|
83
94
|
from swift.common.swob import Request
|
84
95
|
from swift.common.utils import (get_logger, get_remote_client,
|
85
96
|
config_true_value, reiterate,
|
@@ -238,6 +249,7 @@ class ProxyLoggingMiddleware(object):
|
|
238
249
|
:param start_time: timestamp request started
|
239
250
|
:param end_time: timestamp request completed
|
240
251
|
:param resp_headers: dict of the response headers
|
252
|
+
:param ttfb: time to first byte
|
241
253
|
:param wire_status_int: the on the wire status int
|
242
254
|
"""
|
243
255
|
self.obscure_req(req)
|
@@ -260,10 +272,8 @@ class ProxyLoggingMiddleware(object):
|
|
260
272
|
duration_time_str = "%.4f" % (end_time - start_time)
|
261
273
|
policy_index = get_policy_index(req.headers, resp_headers)
|
262
274
|
|
263
|
-
acc, cont, obj = None, None, None
|
264
275
|
swift_path = req.environ.get('swift.backend_path', req.path)
|
265
|
-
|
266
|
-
_, acc, cont, obj = split_path(swift_path, 1, 4, True)
|
276
|
+
acc, cont, obj = self.get_aco_from_path(swift_path)
|
267
277
|
|
268
278
|
replacements = {
|
269
279
|
# Time information
|
@@ -308,8 +318,7 @@ class ProxyLoggingMiddleware(object):
|
|
308
318
|
'transaction_id': req.environ.get('swift.trans_id'),
|
309
319
|
'request_time': duration_time_str,
|
310
320
|
'source': req.environ.get('swift.source'),
|
311
|
-
'log_info':
|
312
|
-
','.join(req.environ.get('swift.log_info', '')),
|
321
|
+
'log_info': get_log_info(req.environ),
|
313
322
|
'policy_index': policy_index,
|
314
323
|
'ttfb': ttfb,
|
315
324
|
'pid': self.pid,
|
@@ -338,22 +347,28 @@ class ProxyLoggingMiddleware(object):
|
|
338
347
|
self.access_logger.update_stats(metric_name_policy + '.xfer',
|
339
348
|
bytes_received + bytes_sent)
|
340
349
|
|
350
|
+
def get_aco_from_path(self, swift_path):
|
351
|
+
try:
|
352
|
+
version, acc, cont, obj = split_path(swift_path, 1, 4, True)
|
353
|
+
if not valid_api_version(version):
|
354
|
+
raise ValueError
|
355
|
+
except ValueError:
|
356
|
+
acc, cont, obj = None, None, None
|
357
|
+
return acc, cont, obj
|
358
|
+
|
341
359
|
def get_metric_name_type(self, req):
|
342
360
|
swift_path = req.environ.get('swift.backend_path', req.path)
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
return stat_type
|
361
|
+
acc, cont, obj = self.get_aco_from_path(swift_path)
|
362
|
+
if obj:
|
363
|
+
return 'object'
|
364
|
+
if cont:
|
365
|
+
return 'container'
|
366
|
+
if acc:
|
367
|
+
return 'account'
|
368
|
+
return req.environ.get('swift.source') or 'UNKNOWN'
|
352
369
|
|
353
370
|
def statsd_metric_name(self, req, status_int, method):
|
354
371
|
stat_type = self.get_metric_name_type(req)
|
355
|
-
if stat_type is None:
|
356
|
-
return None
|
357
372
|
stat_method = method if method in self.valid_methods \
|
358
373
|
else 'BAD_METHOD'
|
359
374
|
return '.'.join((stat_type, stat_method, str(status_int)))
|
@@ -389,11 +404,11 @@ class ProxyLoggingMiddleware(object):
|
|
389
404
|
def my_start_response(status, headers, exc_info=None):
|
390
405
|
start_response_args[0] = (status, list(headers), exc_info)
|
391
406
|
|
392
|
-
def status_int_for_logging(
|
407
|
+
def status_int_for_logging():
|
393
408
|
# log disconnected clients as '499' status code
|
394
|
-
if
|
409
|
+
if input_proxy.client_disconnect:
|
395
410
|
return 499
|
396
|
-
return
|
411
|
+
return env.get('swift.proxy_logging_status')
|
397
412
|
|
398
413
|
def iter_response(iterable):
|
399
414
|
iterator = reiterate(iterable)
|
@@ -438,21 +453,19 @@ class ProxyLoggingMiddleware(object):
|
|
438
453
|
metric_name_policy + '.first-byte.timing', ttfb * 1000)
|
439
454
|
|
440
455
|
bytes_sent = 0
|
441
|
-
client_disconnect = False
|
442
|
-
start_status = wire_status_int
|
443
456
|
try:
|
444
457
|
for chunk in iterator:
|
445
458
|
bytes_sent += len(chunk)
|
446
459
|
yield chunk
|
447
460
|
except GeneratorExit: # generator was closed before we finished
|
448
|
-
|
461
|
+
env['swift.proxy_logging_status'] = 499
|
449
462
|
raise
|
450
463
|
except Exception:
|
451
|
-
|
464
|
+
env['swift.proxy_logging_status'] = 500
|
452
465
|
raise
|
453
466
|
finally:
|
454
|
-
|
455
|
-
|
467
|
+
env.setdefault('swift.proxy_logging_status', wire_status_int)
|
468
|
+
status_int = status_int_for_logging()
|
456
469
|
self.log_request(
|
457
470
|
req, status_int, input_proxy.bytes_received, bytes_sent,
|
458
471
|
start_time, time.time(), resp_headers=resp_headers,
|
@@ -463,7 +476,8 @@ class ProxyLoggingMiddleware(object):
|
|
463
476
|
iterable = self.app(env, my_start_response)
|
464
477
|
except Exception:
|
465
478
|
req = Request(env)
|
466
|
-
|
479
|
+
env['swift.proxy_logging_status'] = 500
|
480
|
+
status_int = status_int_for_logging()
|
467
481
|
self.log_request(
|
468
482
|
req, status_int, input_proxy.bytes_received, 0, start_time,
|
469
483
|
time.time())
|
@@ -13,7 +13,6 @@
|
|
13
13
|
# See the License for the specific language governing permissions and
|
14
14
|
# limitations under the License.
|
15
15
|
import time
|
16
|
-
from swift import gettext_ as _
|
17
16
|
|
18
17
|
import eventlet
|
19
18
|
|
@@ -261,7 +260,7 @@ class RateLimitMiddleware(object):
|
|
261
260
|
|
262
261
|
if account_name in self.ratelimit_blacklist or \
|
263
262
|
account_global_ratelimit == 'BLACKLIST':
|
264
|
-
self.logger.error(
|
263
|
+
self.logger.error('Returning 497 because of blacklisting: %s',
|
265
264
|
account_name)
|
266
265
|
eventlet.sleep(self.BLACK_LIST_SLEEP)
|
267
266
|
return Response(status='497 Blacklisted',
|
@@ -276,8 +275,8 @@ class RateLimitMiddleware(object):
|
|
276
275
|
if self.log_sleep_time_seconds and \
|
277
276
|
need_to_sleep > self.log_sleep_time_seconds:
|
278
277
|
self.logger.warning(
|
279
|
-
|
280
|
-
|
278
|
+
"Ratelimit sleep log: %(sleep)s for "
|
279
|
+
"%(account)s/%(container)s/%(object)s",
|
281
280
|
{'sleep': need_to_sleep, 'account': account_name,
|
282
281
|
'container': container_name, 'object': obj_name})
|
283
282
|
if need_to_sleep > 0:
|
@@ -288,8 +287,8 @@ class RateLimitMiddleware(object):
|
|
288
287
|
else:
|
289
288
|
path = '/'.join((account_name, container_name))
|
290
289
|
self.logger.error(
|
291
|
-
|
292
|
-
|
290
|
+
'Returning 498 for %(meth)s to %(path)s. '
|
291
|
+
'Ratelimit (Max Sleep) %(e)s',
|
293
292
|
{'meth': req.method, 'path': path, 'e': str(e)})
|
294
293
|
error_resp = Response(status='498 Rate Limited',
|
295
294
|
body='Slow down', request=req)
|
@@ -309,7 +308,7 @@ class RateLimitMiddleware(object):
|
|
309
308
|
self.memcache_client = cache_from_env(env)
|
310
309
|
if not self.memcache_client:
|
311
310
|
self.logger.warning(
|
312
|
-
|
311
|
+
'Warning: Cannot ratelimit without a memcached client')
|
313
312
|
return self.app(env, start_response)
|
314
313
|
try:
|
315
314
|
version, account, container, obj = req.split_path(1, 4, True)
|
swift/common/middleware/recon.py
CHANGED
@@ -20,7 +20,6 @@ import time
|
|
20
20
|
from resource import getpagesize
|
21
21
|
|
22
22
|
from swift import __version__ as swiftver
|
23
|
-
from swift import gettext_ as _
|
24
23
|
from swift.common.constraints import check_mount
|
25
24
|
from swift.common.storage_policy import POLICIES
|
26
25
|
from swift.common.swob import Request, Response
|
@@ -89,11 +88,11 @@ class ReconMiddleware(object):
|
|
89
88
|
if err.errno == errno.ENOENT and ignore_missing:
|
90
89
|
pass
|
91
90
|
else:
|
92
|
-
self.logger.exception(
|
91
|
+
self.logger.exception('Error reading recon cache file')
|
93
92
|
except ValueError:
|
94
|
-
self.logger.exception(
|
93
|
+
self.logger.exception('Error parsing recon cache file')
|
95
94
|
except Exception:
|
96
|
-
self.logger.exception(
|
95
|
+
self.logger.exception('Error retrieving recon data')
|
97
96
|
return dict((key, None) for key in cache_keys)
|
98
97
|
|
99
98
|
def get_version(self):
|
@@ -181,7 +180,7 @@ class ReconMiddleware(object):
|
|
181
180
|
try:
|
182
181
|
return {self.devices: os.listdir(self.devices)}
|
183
182
|
except Exception:
|
184
|
-
self.logger.exception(
|
183
|
+
self.logger.exception('Error listing devices')
|
185
184
|
return {self.devices: None}
|
186
185
|
|
187
186
|
def get_updater_info(self, recon_type):
|
@@ -277,7 +276,7 @@ class ReconMiddleware(object):
|
|
277
276
|
except IOError as err:
|
278
277
|
sums[ringfile] = None
|
279
278
|
if err.errno != errno.ENOENT:
|
280
|
-
self.logger.exception(
|
279
|
+
self.logger.exception('Error reading ringfile')
|
281
280
|
return sums
|
282
281
|
|
283
282
|
def get_swift_conf_md5(self):
|
@@ -287,7 +286,7 @@ class ReconMiddleware(object):
|
|
287
286
|
hexsum = md5_hash_for_file(SWIFT_CONF_FILE)
|
288
287
|
except IOError as err:
|
289
288
|
if err.errno != errno.ENOENT:
|
290
|
-
self.logger.exception(
|
289
|
+
self.logger.exception('Error reading swift.conf')
|
291
290
|
return {SWIFT_CONF_FILE: hexsum}
|
292
291
|
|
293
292
|
def get_quarantine_count(self):
|
@@ -133,6 +133,15 @@ class BaseAclHandler(object):
|
|
133
133
|
query = {}
|
134
134
|
else:
|
135
135
|
query = {'version-id': version_id}
|
136
|
+
if self.req.method == 'HEAD':
|
137
|
+
# This HEAD for ACL is going to also be the definitive response
|
138
|
+
# to the client so we need to include client params. We don't
|
139
|
+
# do this for other client request methods because they may
|
140
|
+
# have invalid combinations of params and headers for a swift
|
141
|
+
# HEAD request.
|
142
|
+
part_number = self.req.params.get('partNumber')
|
143
|
+
if part_number is not None:
|
144
|
+
query['part-number'] = part_number
|
136
145
|
resp = self.req.get_acl_response(app, 'HEAD',
|
137
146
|
container, obj,
|
138
147
|
headers, query=query)
|
@@ -168,7 +177,7 @@ class BaseAclHandler(object):
|
|
168
177
|
elem = fromstring(body, ACL.root_tag)
|
169
178
|
acl = ACL.from_elem(
|
170
179
|
elem, True, self.req.conf.allow_no_owner)
|
171
|
-
except(XMLSyntaxError, DocumentInvalid):
|
180
|
+
except (XMLSyntaxError, DocumentInvalid):
|
172
181
|
raise MalformedACLError()
|
173
182
|
except Exception as e:
|
174
183
|
self.logger.error(e)
|
@@ -33,6 +33,8 @@ from swift.common.middleware.s3api.controllers.versioning import \
|
|
33
33
|
VersioningController
|
34
34
|
from swift.common.middleware.s3api.controllers.tagging import \
|
35
35
|
TaggingController
|
36
|
+
from swift.common.middleware.s3api.controllers.object_lock import \
|
37
|
+
ObjectLockController
|
36
38
|
|
37
39
|
__all__ = [
|
38
40
|
'Controller',
|
@@ -50,6 +52,7 @@ __all__ = [
|
|
50
52
|
'LoggingStatusController',
|
51
53
|
'VersioningController',
|
52
54
|
'TaggingController',
|
55
|
+
'ObjectLockController',
|
53
56
|
|
54
57
|
'UnsupportedController',
|
55
58
|
]
|
@@ -19,8 +19,9 @@ from swift.common.utils import public
|
|
19
19
|
|
20
20
|
from swift.common.middleware.s3api.exception import ACLError
|
21
21
|
from swift.common.middleware.s3api.controllers.base import Controller
|
22
|
-
from swift.common.middleware.s3api.s3response import
|
23
|
-
MalformedACLError, UnexpectedContent,
|
22
|
+
from swift.common.middleware.s3api.s3response import (
|
23
|
+
HTTPOk, S3NotImplemented, MalformedACLError, UnexpectedContent,
|
24
|
+
MissingSecurityHeader)
|
24
25
|
from swift.common.middleware.s3api.etree import Element, SubElement, tostring
|
25
26
|
from swift.common.middleware.s3api.acl_utils import swift_acl_translate, \
|
26
27
|
XMLNS_XSI
|
@@ -18,8 +18,8 @@ from swift.common.utils import public
|
|
18
18
|
from swift.common.middleware.s3api.controllers.base import Controller, \
|
19
19
|
bucket_operation
|
20
20
|
from swift.common.middleware.s3api.etree import Element, tostring
|
21
|
-
from swift.common.middleware.s3api.s3response import
|
22
|
-
NoLoggingStatusForKey
|
21
|
+
from swift.common.middleware.s3api.s3response import (
|
22
|
+
HTTPOk, S3NotImplemented, NoLoggingStatusForKey)
|
23
23
|
|
24
24
|
|
25
25
|
class LoggingStatusController(Controller):
|
@@ -68,8 +68,9 @@ import time
|
|
68
68
|
import six
|
69
69
|
|
70
70
|
from swift.common import constraints
|
71
|
-
from swift.common.swob import Range, bytes_to_wsgi, normalize_etag,
|
72
|
-
|
71
|
+
from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, \
|
72
|
+
wsgi_to_str
|
73
|
+
from swift.common.utils import json, public, reiterate, md5, Timestamp
|
73
74
|
from swift.common.db import utf8encode
|
74
75
|
from swift.common.request_helpers import get_container_update_override_key, \
|
75
76
|
get_param
|
@@ -82,7 +83,7 @@ from swift.common.middleware.s3api.s3response import InvalidArgument, \
|
|
82
83
|
ErrorResponse, MalformedXML, BadDigest, KeyTooLongError, \
|
83
84
|
InvalidPart, BucketAlreadyExists, EntityTooSmall, InvalidPartOrder, \
|
84
85
|
InvalidRequest, HTTPOk, HTTPNoContent, NoSuchKey, NoSuchUpload, \
|
85
|
-
NoSuchBucket, BucketAlreadyOwnedByYou
|
86
|
+
NoSuchBucket, BucketAlreadyOwnedByYou, ServiceUnavailable
|
86
87
|
from swift.common.middleware.s3api.utils import unique_id, \
|
87
88
|
MULTIUPLOAD_SUFFIX, S3Timestamp, sysmeta_header
|
88
89
|
from swift.common.middleware.s3api.etree import Element, SubElement, \
|
@@ -96,6 +97,17 @@ MAX_COMPLETE_UPLOAD_BODY_SIZE = 2048 * 1024
|
|
96
97
|
|
97
98
|
|
98
99
|
def _get_upload_info(req, app, upload_id):
|
100
|
+
"""
|
101
|
+
Make a HEAD request for existing upload object metadata. Tries the upload
|
102
|
+
marker first, and then falls back to the manifest object.
|
103
|
+
|
104
|
+
:param req: an S3Request object.
|
105
|
+
:param app: the wsgi app.
|
106
|
+
:param upload_id: the upload id.
|
107
|
+
:returns: a tuple of (S3Response, boolean) where the boolean is True if the
|
108
|
+
response is from the upload marker and False otherwise.
|
109
|
+
:raises: NoSuchUpload if neither the marker nor the manifest were found.
|
110
|
+
"""
|
99
111
|
|
100
112
|
container = req.container_name + MULTIUPLOAD_SUFFIX
|
101
113
|
obj = '%s/%s' % (req.object_name, upload_id)
|
@@ -106,14 +118,17 @@ def _get_upload_info(req, app, upload_id):
|
|
106
118
|
# it off for now...
|
107
119
|
copy_source = req.headers.pop('X-Amz-Copy-Source', None)
|
108
120
|
try:
|
109
|
-
|
121
|
+
resp = req.get_response(app, 'HEAD', container=container, obj=obj)
|
122
|
+
return resp, True
|
110
123
|
except NoSuchKey:
|
124
|
+
# ensure consistent path and policy are logged despite manifest HEAD
|
111
125
|
upload_marker_path = req.environ.get('s3api.backend_path')
|
126
|
+
policy_index = req.policy_index
|
112
127
|
try:
|
113
128
|
resp = req.get_response(app, 'HEAD')
|
114
129
|
if resp.sysmeta_headers.get(sysmeta_header(
|
115
130
|
'object', 'upload-id')) == upload_id:
|
116
|
-
return resp
|
131
|
+
return resp, False
|
117
132
|
except NoSuchKey:
|
118
133
|
pass
|
119
134
|
finally:
|
@@ -121,6 +136,8 @@ def _get_upload_info(req, app, upload_id):
|
|
121
136
|
# path, so put it back
|
122
137
|
if upload_marker_path is not None:
|
123
138
|
req.environ['s3api.backend_path'] = upload_marker_path
|
139
|
+
if policy_index is not None:
|
140
|
+
req.policy_index = policy_index
|
124
141
|
raise NoSuchUpload(upload_id=upload_id)
|
125
142
|
finally:
|
126
143
|
# ...making sure to restore any copy-source before returning
|
@@ -179,15 +196,7 @@ class PartController(Controller):
|
|
179
196
|
raise InvalidArgument('ResourceType', 'partNumber',
|
180
197
|
'Unexpected query string parameter')
|
181
198
|
|
182
|
-
|
183
|
-
part_number = int(get_param(req, 'partNumber'))
|
184
|
-
if part_number < 1 or self.conf.max_upload_part_num < part_number:
|
185
|
-
raise Exception()
|
186
|
-
except Exception:
|
187
|
-
err_msg = 'Part number must be an integer between 1 and %d,' \
|
188
|
-
' inclusive' % self.conf.max_upload_part_num
|
189
|
-
raise InvalidArgument('partNumber', get_param(req, 'partNumber'),
|
190
|
-
err_msg)
|
199
|
+
part_number = req.validate_part_number()
|
191
200
|
|
192
201
|
upload_id = get_param(req, 'uploadId')
|
193
202
|
_get_upload_info(req, self.app, upload_id)
|
@@ -651,7 +660,14 @@ class UploadController(Controller):
|
|
651
660
|
Handles Complete Multipart Upload.
|
652
661
|
"""
|
653
662
|
upload_id = get_param(req, 'uploadId')
|
654
|
-
resp = _get_upload_info(req, self.app, upload_id)
|
663
|
+
resp, is_marker = _get_upload_info(req, self.app, upload_id)
|
664
|
+
if (is_marker and
|
665
|
+
resp.sw_headers.get('X-Backend-Timestamp') >= Timestamp.now()):
|
666
|
+
# Somehow the marker was created in the future w.r.t. this thread's
|
667
|
+
# clock. The manifest PUT may succeed but the subsequent marker
|
668
|
+
# DELETE will fail, so don't attempt either.
|
669
|
+
raise ServiceUnavailable
|
670
|
+
|
655
671
|
headers = {'Accept': 'application/json',
|
656
672
|
sysmeta_header('object', 'upload-id'): upload_id}
|
657
673
|
for key, val in resp.headers.items():
|
@@ -90,8 +90,14 @@ class ObjectController(Controller):
|
|
90
90
|
if version_id not in ('null', None) and \
|
91
91
|
'object_versioning' not in get_swift_info():
|
92
92
|
raise S3NotImplemented()
|
93
|
+
part_number = req.validate_part_number(check_max=False)
|
94
|
+
|
95
|
+
query = {}
|
96
|
+
if version_id is not None:
|
97
|
+
query['version-id'] = version_id
|
98
|
+
if part_number is not None:
|
99
|
+
query['part-number'] = part_number
|
93
100
|
|
94
|
-
query = {} if version_id is None else {'version-id': version_id}
|
95
101
|
if version_id not in ('null', None):
|
96
102
|
container_info = req.get_container_info(self.app)
|
97
103
|
if not container_info.get(
|
@@ -101,6 +107,19 @@ class ObjectController(Controller):
|
|
101
107
|
|
102
108
|
resp = req.get_response(self.app, query=query)
|
103
109
|
|
110
|
+
if not resp.is_slo:
|
111
|
+
# SLO ignores part_number for non-slo objects, but s3api only
|
112
|
+
# allows the query param for non-MPU if it's exactly 1.
|
113
|
+
part_number = req.validate_part_number(parts_count=1)
|
114
|
+
if part_number == 1:
|
115
|
+
# When the query param *is* exactly 1 the response status code
|
116
|
+
# and headers are updated.
|
117
|
+
resp.status = HTTP_PARTIAL_CONTENT
|
118
|
+
resp.headers['Content-Range'] = \
|
119
|
+
'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1,
|
120
|
+
resp.headers['Content-Length'])
|
121
|
+
# else: part_number is None
|
122
|
+
|
104
123
|
if req.method == 'HEAD':
|
105
124
|
resp.app_iter = None
|
106
125
|
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# Copyright (c) 2010-2023 OpenStack Foundation
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
12
|
+
# implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
from swift.common.utils import public
|
17
|
+
|
18
|
+
from swift.common.middleware.s3api.controllers.base import Controller, \
|
19
|
+
bucket_operation, S3NotImplemented
|
20
|
+
from swift.common.middleware.s3api.s3response import \
|
21
|
+
ObjectLockConfigurationNotFoundError
|
22
|
+
|
23
|
+
|
24
|
+
class ObjectLockController(Controller):
|
25
|
+
"""
|
26
|
+
Handles GET object-lock request, which always returns
|
27
|
+
<ObjectLockEnabled>Disabled</ObjectLockEnabled>
|
28
|
+
"""
|
29
|
+
@public
|
30
|
+
@bucket_operation
|
31
|
+
def GET(self, req):
|
32
|
+
"""
|
33
|
+
Handles GET object-lock param calls.
|
34
|
+
"""
|
35
|
+
raise ObjectLockConfigurationNotFoundError(req.container_name)
|
36
|
+
|
37
|
+
@public
|
38
|
+
@bucket_operation
|
39
|
+
def PUT(self, req):
|
40
|
+
"""
|
41
|
+
Handles PUT object-lock param calls.
|
42
|
+
"""
|
43
|
+
# Basically we don't support it, so return a 501
|
44
|
+
raise S3NotImplemented('The requested resource is not implemented')
|
@@ -149,6 +149,7 @@ from six.moves.urllib.parse import parse_qs
|
|
149
149
|
from swift.common.constraints import valid_api_version
|
150
150
|
from swift.common.middleware.listing_formats import \
|
151
151
|
MAX_CONTAINER_LISTING_CONTENT_LENGTH
|
152
|
+
from swift.common.request_helpers import append_log_info
|
152
153
|
from swift.common.wsgi import PipelineWrapper, loadcontext, WSGIContext
|
153
154
|
|
154
155
|
from swift.common.middleware import app_property
|
@@ -353,6 +354,7 @@ class S3ApiMiddleware(object):
|
|
353
354
|
self.logger.debug(e.cause)
|
354
355
|
except ErrorResponse as err_resp:
|
355
356
|
self.logger.increment(err_resp.metric_name)
|
357
|
+
append_log_info(env, 's3:err:%s' % err_resp.summary)
|
356
358
|
if isinstance(err_resp, InternalError):
|
357
359
|
self.logger.exception(err_resp)
|
358
360
|
resp = err_resp
|
@@ -366,6 +368,7 @@ class S3ApiMiddleware(object):
|
|
366
368
|
|
367
369
|
if 's3api.backend_path' in env and 'swift.backend_path' not in env:
|
368
370
|
env['swift.backend_path'] = env['s3api.backend_path']
|
371
|
+
|
369
372
|
return resp(env, start_response)
|
370
373
|
|
371
374
|
def handle_request(self, req):
|
@@ -390,6 +393,9 @@ class S3ApiMiddleware(object):
|
|
390
393
|
raise MethodNotAllowed(req.method,
|
391
394
|
req.controller.resource_type())
|
392
395
|
|
396
|
+
if req.policy_index is not None:
|
397
|
+
res.headers.setdefault('X-Backend-Storage-Policy-Index',
|
398
|
+
req.policy_index)
|
393
399
|
return res
|
394
400
|
|
395
401
|
def check_pipeline(self, wsgi_conf):
|