swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. swift/account/auditor.py +11 -0
  2. swift/account/reaper.py +11 -1
  3. swift/account/replicator.py +22 -0
  4. swift/account/server.py +13 -12
  5. swift-2.32.0.data/scripts/swift-account-audit → swift/cli/account_audit.py +6 -2
  6. swift-2.32.0.data/scripts/swift-config → swift/cli/config.py +1 -1
  7. swift-2.32.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +6 -2
  8. swift-2.32.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +12 -3
  9. swift-2.32.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +6 -2
  10. swift/cli/info.py +131 -3
  11. swift-2.32.0.data/scripts/swift-oldies → swift/cli/oldies.py +6 -3
  12. swift-2.32.0.data/scripts/swift-orphans → swift/cli/orphans.py +7 -2
  13. swift-2.32.0.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +9 -18
  14. swift-2.32.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  15. swift/cli/relinker.py +1 -1
  16. swift/cli/reload.py +141 -0
  17. swift/cli/ringbuilder.py +24 -0
  18. swift/common/daemon.py +12 -2
  19. swift/common/db.py +14 -9
  20. swift/common/db_auditor.py +2 -2
  21. swift/common/db_replicator.py +6 -0
  22. swift/common/exceptions.py +12 -0
  23. swift/common/http_protocol.py +76 -3
  24. swift/common/manager.py +120 -5
  25. swift/common/memcached.py +24 -25
  26. swift/common/middleware/account_quotas.py +144 -43
  27. swift/common/middleware/backend_ratelimit.py +166 -24
  28. swift/common/middleware/catch_errors.py +1 -3
  29. swift/common/middleware/cname_lookup.py +3 -5
  30. swift/common/middleware/container_sync.py +6 -10
  31. swift/common/middleware/crypto/crypto_utils.py +4 -5
  32. swift/common/middleware/crypto/decrypter.py +4 -5
  33. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  34. swift/common/middleware/proxy_logging.py +57 -43
  35. swift/common/middleware/ratelimit.py +6 -7
  36. swift/common/middleware/recon.py +6 -7
  37. swift/common/middleware/s3api/acl_handlers.py +10 -1
  38. swift/common/middleware/s3api/controllers/__init__.py +3 -0
  39. swift/common/middleware/s3api/controllers/acl.py +3 -2
  40. swift/common/middleware/s3api/controllers/logging.py +2 -2
  41. swift/common/middleware/s3api/controllers/multi_upload.py +31 -15
  42. swift/common/middleware/s3api/controllers/obj.py +20 -1
  43. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  44. swift/common/middleware/s3api/s3api.py +6 -0
  45. swift/common/middleware/s3api/s3request.py +190 -74
  46. swift/common/middleware/s3api/s3response.py +48 -8
  47. swift/common/middleware/s3api/s3token.py +2 -2
  48. swift/common/middleware/s3api/utils.py +2 -1
  49. swift/common/middleware/slo.py +508 -310
  50. swift/common/middleware/staticweb.py +45 -14
  51. swift/common/middleware/tempauth.py +6 -4
  52. swift/common/middleware/tempurl.py +134 -93
  53. swift/common/middleware/x_profile/exceptions.py +1 -4
  54. swift/common/middleware/x_profile/html_viewer.py +9 -10
  55. swift/common/middleware/x_profile/profile_model.py +1 -2
  56. swift/common/middleware/xprofile.py +1 -2
  57. swift/common/request_helpers.py +101 -8
  58. swift/common/statsd_client.py +207 -0
  59. swift/common/storage_policy.py +1 -1
  60. swift/common/swob.py +5 -2
  61. swift/common/utils/__init__.py +331 -1774
  62. swift/common/utils/base.py +138 -0
  63. swift/common/utils/config.py +443 -0
  64. swift/common/utils/logs.py +999 -0
  65. swift/common/utils/timestamp.py +23 -2
  66. swift/common/wsgi.py +19 -3
  67. swift/container/auditor.py +11 -0
  68. swift/container/backend.py +136 -31
  69. swift/container/reconciler.py +11 -2
  70. swift/container/replicator.py +64 -7
  71. swift/container/server.py +276 -146
  72. swift/container/sharder.py +86 -42
  73. swift/container/sync.py +11 -1
  74. swift/container/updater.py +12 -2
  75. swift/obj/auditor.py +20 -3
  76. swift/obj/diskfile.py +63 -25
  77. swift/obj/expirer.py +154 -47
  78. swift/obj/mem_diskfile.py +2 -1
  79. swift/obj/mem_server.py +1 -0
  80. swift/obj/reconstructor.py +28 -4
  81. swift/obj/replicator.py +63 -24
  82. swift/obj/server.py +76 -59
  83. swift/obj/updater.py +12 -2
  84. swift/obj/watchers/dark_data.py +72 -34
  85. swift/proxy/controllers/account.py +3 -2
  86. swift/proxy/controllers/base.py +254 -148
  87. swift/proxy/controllers/container.py +274 -289
  88. swift/proxy/controllers/obj.py +120 -166
  89. swift/proxy/server.py +17 -13
  90. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/AUTHORS +14 -4
  91. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/METADATA +9 -7
  92. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/RECORD +97 -120
  93. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/entry_points.txt +39 -0
  94. swift-2.34.0.dist-info/pbr.json +1 -0
  95. swift-2.32.0.data/scripts/swift-account-auditor +0 -23
  96. swift-2.32.0.data/scripts/swift-account-info +0 -52
  97. swift-2.32.0.data/scripts/swift-account-reaper +0 -23
  98. swift-2.32.0.data/scripts/swift-account-replicator +0 -34
  99. swift-2.32.0.data/scripts/swift-account-server +0 -23
  100. swift-2.32.0.data/scripts/swift-container-auditor +0 -23
  101. swift-2.32.0.data/scripts/swift-container-info +0 -56
  102. swift-2.32.0.data/scripts/swift-container-reconciler +0 -21
  103. swift-2.32.0.data/scripts/swift-container-replicator +0 -34
  104. swift-2.32.0.data/scripts/swift-container-server +0 -23
  105. swift-2.32.0.data/scripts/swift-container-sharder +0 -37
  106. swift-2.32.0.data/scripts/swift-container-sync +0 -23
  107. swift-2.32.0.data/scripts/swift-container-updater +0 -23
  108. swift-2.32.0.data/scripts/swift-dispersion-report +0 -24
  109. swift-2.32.0.data/scripts/swift-form-signature +0 -20
  110. swift-2.32.0.data/scripts/swift-init +0 -119
  111. swift-2.32.0.data/scripts/swift-object-auditor +0 -29
  112. swift-2.32.0.data/scripts/swift-object-expirer +0 -33
  113. swift-2.32.0.data/scripts/swift-object-info +0 -60
  114. swift-2.32.0.data/scripts/swift-object-reconstructor +0 -33
  115. swift-2.32.0.data/scripts/swift-object-relinker +0 -23
  116. swift-2.32.0.data/scripts/swift-object-replicator +0 -37
  117. swift-2.32.0.data/scripts/swift-object-server +0 -27
  118. swift-2.32.0.data/scripts/swift-object-updater +0 -23
  119. swift-2.32.0.data/scripts/swift-proxy-server +0 -23
  120. swift-2.32.0.data/scripts/swift-recon +0 -24
  121. swift-2.32.0.data/scripts/swift-ring-builder +0 -37
  122. swift-2.32.0.data/scripts/swift-ring-builder-analyzer +0 -22
  123. swift-2.32.0.data/scripts/swift-ring-composer +0 -22
  124. swift-2.32.0.dist-info/pbr.json +0 -1
  125. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/LICENSE +0 -0
  126. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/WHEEL +0 -0
  127. {swift-2.32.0.dist-info → swift-2.34.0.dist-info}/top_level.txt +0 -0
@@ -22,7 +22,6 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
22
22
  import six
23
23
  from six.moves.urllib import parse as urlparse
24
24
 
25
- from swift import gettext_ as _
26
25
  from swift.common.exceptions import EncryptionException, UnknownSecretIdError
27
26
  from swift.common.swob import HTTPInternalServerError
28
27
  from swift.common.utils import get_logger
@@ -160,7 +159,7 @@ class CryptoWSGIContext(WSGIContext):
160
159
  try:
161
160
  fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
162
161
  except KeyError:
163
- self.logger.exception(_('ERROR get_keys() missing callback'))
162
+ self.logger.exception('ERROR get_keys() missing callback')
164
163
  raise HTTPInternalServerError(
165
164
  "Unable to retrieve encryption keys.")
166
165
 
@@ -181,12 +180,12 @@ class CryptoWSGIContext(WSGIContext):
181
180
  self.crypto.check_key(key)
182
181
  continue
183
182
  except KeyError:
184
- self.logger.exception(_("Missing key for %r") % name)
183
+ self.logger.exception("Missing key for %r", name)
185
184
  except TypeError:
186
- self.logger.exception(_("Did not get a keys dict"))
185
+ self.logger.exception("Did not get a keys dict")
187
186
  except ValueError as e:
188
187
  # don't include the key in any messages!
189
- self.logger.exception(_("Bad key for %(name)r: %(err)s") %
188
+ self.logger.exception("Bad key for %(name)r: %(err)s",
190
189
  {'name': name, 'err': e})
191
190
  raise HTTPInternalServerError(
192
191
  "Unable to retrieve encryption keys.")
@@ -16,7 +16,6 @@
16
16
  import base64
17
17
  import json
18
18
 
19
- from swift import gettext_ as _
20
19
  from swift.common.header_key_dict import HeaderKeyDict
21
20
  from swift.common.http import is_success
22
21
  from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
@@ -77,10 +76,10 @@ class BaseDecrypterContext(CryptoWSGIContext):
77
76
  crypto_meta['body_key'])
78
77
  except KeyError as err:
79
78
  self.logger.error(
80
- _('Error decrypting %(resp_type)s: Missing %(key)s'),
79
+ 'Error decrypting %(resp_type)s: Missing %(key)s',
81
80
  {'resp_type': self.server_type, 'key': err})
82
81
  except ValueError as err:
83
- self.logger.error(_('Error decrypting %(resp_type)s: %(reason)s'),
82
+ self.logger.error('Error decrypting %(resp_type)s: %(reason)s',
84
83
  {'resp_type': self.server_type, 'reason': err})
85
84
  raise HTTPInternalServerError(
86
85
  body='Error decrypting %s' % self.server_type,
@@ -178,7 +177,7 @@ class DecrypterObjContext(BaseDecrypterContext):
178
177
  value, key, required, bytes_to_wsgi)
179
178
  except EncryptionException as err:
180
179
  self.logger.error(
181
- _("Error decrypting header %(header)s: %(error)s"),
180
+ "Error decrypting header %(header)s: %(error)s",
182
181
  {'header': header, 'error': err})
183
182
  raise HTTPInternalServerError(
184
183
  body='Error decrypting header',
@@ -313,7 +312,7 @@ class DecrypterObjContext(BaseDecrypterContext):
313
312
  try:
314
313
  crypto_meta = self.get_crypto_meta(header, check)
315
314
  except EncryptionException as err:
316
- self.logger.error(_('Error decrypting object: %s'), err)
315
+ self.logger.error('Error decrypting object: %s', err)
317
316
  raise HTTPInternalServerError(
318
317
  body='Error decrypting object', content_type='text/plain')
319
318
  return crypto_meta
@@ -34,7 +34,7 @@ class KmsKeyMaster(BaseKeyMaster):
34
34
  'domain_id', 'domain_name', 'project_id',
35
35
  'project_domain_id', 'reauthenticate',
36
36
  'auth_endpoint', 'api_class', 'key_id*',
37
- 'active_root_secret_id')
37
+ 'barbican_endpoint', 'active_root_secret_id')
38
38
  keymaster_conf_section = 'kms_keymaster'
39
39
 
40
40
  def _get_root_secret(self, conf):
@@ -67,6 +67,7 @@ class KmsKeyMaster(BaseKeyMaster):
67
67
  oslo_conf = cfg.ConfigOpts()
68
68
  options.set_defaults(
69
69
  oslo_conf, auth_endpoint=conf.get('auth_endpoint'),
70
+ barbican_endpoint=conf.get('barbican_endpoint'),
70
71
  api_class=conf.get('api_class')
71
72
  )
72
73
  options.enable_logging()
@@ -27,19 +27,28 @@ The logging format implemented below is as follows::
27
27
  start_time end_time policy_index
28
28
 
29
29
  These values are space-separated, and each is url-encoded, so that they can
30
- be separated with a simple .split()
31
-
32
- * remote_addr is the contents of the REMOTE_ADDR environment variable, while
33
- client_ip is swift's best guess at the end-user IP, extracted variously
34
- from the X-Forwarded-For header, X-Cluster-Ip header, or the REMOTE_ADDR
35
- environment variable.
36
-
37
- * source (swift.source in the WSGI environment) indicates the code
30
+ be separated with a simple ``.split()``.
31
+
32
+ * ``remote_addr`` is the contents of the REMOTE_ADDR environment variable,
33
+ while ``client_ip`` is swift's best guess at the end-user IP, extracted
34
+ variously from the X-Forwarded-For header, X-Cluster-Ip header, or the
35
+ REMOTE_ADDR environment variable.
36
+
37
+ * ``status_int`` is the integer part of the ``status`` string passed to this
38
+ middleware's start_response function, unless the WSGI environment has an item
39
+ with key ``swift.proxy_logging_status``, in which case the value of that item
40
+ is used. Other middleware's may set ``swift.proxy_logging_status`` to
41
+ override the logging of ``status_int``. In either case, the logged
42
+ ``status_int`` value is forced to 499 if a client disconnect is detected
43
+ while this middleware is handling a request, or 500 if an exception is caught
44
+ while handling a request.
45
+
46
+ * ``source`` (``swift.source`` in the WSGI environment) indicates the code
38
47
  that generated the request, such as most middleware. (See below for
39
48
  more detail.)
40
49
 
41
- * log_info (swift.log_info in the WSGI environment) is for additional
42
- information that could prove quite useful, such as any x-delete-at
50
+ * ``log_info`` (``swift.log_info`` in the WSGI environment) is for additional
51
+ information that could prove quite useful, such as any ``x-delete-at``
43
52
  value or other "behind the scenes" activity that might not
44
53
  otherwise be detectable from the plain log information. Code that
45
54
  wishes to add additional log information should use code like
@@ -62,24 +71,26 @@ For example, with staticweb, the middleware might intercept a request to
62
71
  /v1/AUTH_acc/cont/, make a subrequest to the proxy to retrieve
63
72
  /v1/AUTH_acc/cont/index.html and, in effect, respond to the client's original
64
73
  request using the 2nd request's body. In this instance the subrequest will be
65
- logged by the rightmost middleware (with a swift.source set) and the outgoing
66
- request (with body overridden) will be logged by leftmost middleware.
74
+ logged by the rightmost middleware (with a ``swift.source`` set) and the
75
+ outgoing request (with body overridden) will be logged by leftmost middleware.
67
76
 
68
77
  Requests that follow the normal pipeline (use the same wsgi environment
69
78
  throughout) will not be double logged because an environment variable
70
- (swift.proxy_access_log_made) is checked/set when a log is made.
79
+ (``swift.proxy_access_log_made``) is checked/set when a log is made.
71
80
 
72
- All middleware making subrequests should take care to set swift.source when
81
+ All middleware making subrequests should take care to set ``swift.source`` when
73
82
  needed. With the doubled proxy logs, any consumer/processor of swift's proxy
74
- logs should look at the swift.source field, the rightmost log value, to decide
75
- if this is a middleware subrequest or not. A log processor calculating
76
- bandwidth usage will want to only sum up logs with no swift.source.
83
+ logs should look at the ``swift.source`` field, the rightmost log value, to
84
+ decide if this is a middleware subrequest or not. A log processor calculating
85
+ bandwidth usage will want to only sum up logs with no ``swift.source``.
77
86
  """
78
87
 
79
88
  import os
80
89
  import time
81
90
 
91
+ from swift.common.constraints import valid_api_version
82
92
  from swift.common.middleware.catch_errors import enforce_byte_count
93
+ from swift.common.request_helpers import get_log_info
83
94
  from swift.common.swob import Request
84
95
  from swift.common.utils import (get_logger, get_remote_client,
85
96
  config_true_value, reiterate,
@@ -238,6 +249,7 @@ class ProxyLoggingMiddleware(object):
238
249
  :param start_time: timestamp request started
239
250
  :param end_time: timestamp request completed
240
251
  :param resp_headers: dict of the response headers
252
+ :param ttfb: time to first byte
241
253
  :param wire_status_int: the on the wire status int
242
254
  """
243
255
  self.obscure_req(req)
@@ -260,10 +272,8 @@ class ProxyLoggingMiddleware(object):
260
272
  duration_time_str = "%.4f" % (end_time - start_time)
261
273
  policy_index = get_policy_index(req.headers, resp_headers)
262
274
 
263
- acc, cont, obj = None, None, None
264
275
  swift_path = req.environ.get('swift.backend_path', req.path)
265
- if swift_path.startswith('/v1/'):
266
- _, acc, cont, obj = split_path(swift_path, 1, 4, True)
276
+ acc, cont, obj = self.get_aco_from_path(swift_path)
267
277
 
268
278
  replacements = {
269
279
  # Time information
@@ -308,8 +318,7 @@ class ProxyLoggingMiddleware(object):
308
318
  'transaction_id': req.environ.get('swift.trans_id'),
309
319
  'request_time': duration_time_str,
310
320
  'source': req.environ.get('swift.source'),
311
- 'log_info':
312
- ','.join(req.environ.get('swift.log_info', '')),
321
+ 'log_info': get_log_info(req.environ),
313
322
  'policy_index': policy_index,
314
323
  'ttfb': ttfb,
315
324
  'pid': self.pid,
@@ -338,22 +347,28 @@ class ProxyLoggingMiddleware(object):
338
347
  self.access_logger.update_stats(metric_name_policy + '.xfer',
339
348
  bytes_received + bytes_sent)
340
349
 
350
+ def get_aco_from_path(self, swift_path):
351
+ try:
352
+ version, acc, cont, obj = split_path(swift_path, 1, 4, True)
353
+ if not valid_api_version(version):
354
+ raise ValueError
355
+ except ValueError:
356
+ acc, cont, obj = None, None, None
357
+ return acc, cont, obj
358
+
341
359
  def get_metric_name_type(self, req):
342
360
  swift_path = req.environ.get('swift.backend_path', req.path)
343
- if swift_path.startswith('/v1/'):
344
- try:
345
- stat_type = [None, 'account', 'container',
346
- 'object'][swift_path.strip('/').count('/')]
347
- except IndexError:
348
- stat_type = 'object'
349
- else:
350
- stat_type = req.environ.get('swift.source')
351
- return stat_type
361
+ acc, cont, obj = self.get_aco_from_path(swift_path)
362
+ if obj:
363
+ return 'object'
364
+ if cont:
365
+ return 'container'
366
+ if acc:
367
+ return 'account'
368
+ return req.environ.get('swift.source') or 'UNKNOWN'
352
369
 
353
370
  def statsd_metric_name(self, req, status_int, method):
354
371
  stat_type = self.get_metric_name_type(req)
355
- if stat_type is None:
356
- return None
357
372
  stat_method = method if method in self.valid_methods \
358
373
  else 'BAD_METHOD'
359
374
  return '.'.join((stat_type, stat_method, str(status_int)))
@@ -389,11 +404,11 @@ class ProxyLoggingMiddleware(object):
389
404
  def my_start_response(status, headers, exc_info=None):
390
405
  start_response_args[0] = (status, list(headers), exc_info)
391
406
 
392
- def status_int_for_logging(start_status, client_disconnect=False):
407
+ def status_int_for_logging():
393
408
  # log disconnected clients as '499' status code
394
- if client_disconnect or input_proxy.client_disconnect:
409
+ if input_proxy.client_disconnect:
395
410
  return 499
396
- return start_status
411
+ return env.get('swift.proxy_logging_status')
397
412
 
398
413
  def iter_response(iterable):
399
414
  iterator = reiterate(iterable)
@@ -438,21 +453,19 @@ class ProxyLoggingMiddleware(object):
438
453
  metric_name_policy + '.first-byte.timing', ttfb * 1000)
439
454
 
440
455
  bytes_sent = 0
441
- client_disconnect = False
442
- start_status = wire_status_int
443
456
  try:
444
457
  for chunk in iterator:
445
458
  bytes_sent += len(chunk)
446
459
  yield chunk
447
460
  except GeneratorExit: # generator was closed before we finished
448
- client_disconnect = True
461
+ env['swift.proxy_logging_status'] = 499
449
462
  raise
450
463
  except Exception:
451
- start_status = 500
464
+ env['swift.proxy_logging_status'] = 500
452
465
  raise
453
466
  finally:
454
- status_int = status_int_for_logging(
455
- start_status, client_disconnect)
467
+ env.setdefault('swift.proxy_logging_status', wire_status_int)
468
+ status_int = status_int_for_logging()
456
469
  self.log_request(
457
470
  req, status_int, input_proxy.bytes_received, bytes_sent,
458
471
  start_time, time.time(), resp_headers=resp_headers,
@@ -463,7 +476,8 @@ class ProxyLoggingMiddleware(object):
463
476
  iterable = self.app(env, my_start_response)
464
477
  except Exception:
465
478
  req = Request(env)
466
- status_int = status_int_for_logging(500)
479
+ env['swift.proxy_logging_status'] = 500
480
+ status_int = status_int_for_logging()
467
481
  self.log_request(
468
482
  req, status_int, input_proxy.bytes_received, 0, start_time,
469
483
  time.time())
@@ -13,7 +13,6 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
  import time
16
- from swift import gettext_ as _
17
16
 
18
17
  import eventlet
19
18
 
@@ -261,7 +260,7 @@ class RateLimitMiddleware(object):
261
260
 
262
261
  if account_name in self.ratelimit_blacklist or \
263
262
  account_global_ratelimit == 'BLACKLIST':
264
- self.logger.error(_('Returning 497 because of blacklisting: %s'),
263
+ self.logger.error('Returning 497 because of blacklisting: %s',
265
264
  account_name)
266
265
  eventlet.sleep(self.BLACK_LIST_SLEEP)
267
266
  return Response(status='497 Blacklisted',
@@ -276,8 +275,8 @@ class RateLimitMiddleware(object):
276
275
  if self.log_sleep_time_seconds and \
277
276
  need_to_sleep > self.log_sleep_time_seconds:
278
277
  self.logger.warning(
279
- _("Ratelimit sleep log: %(sleep)s for "
280
- "%(account)s/%(container)s/%(object)s"),
278
+ "Ratelimit sleep log: %(sleep)s for "
279
+ "%(account)s/%(container)s/%(object)s",
281
280
  {'sleep': need_to_sleep, 'account': account_name,
282
281
  'container': container_name, 'object': obj_name})
283
282
  if need_to_sleep > 0:
@@ -288,8 +287,8 @@ class RateLimitMiddleware(object):
288
287
  else:
289
288
  path = '/'.join((account_name, container_name))
290
289
  self.logger.error(
291
- _('Returning 498 for %(meth)s to %(path)s. '
292
- 'Ratelimit (Max Sleep) %(e)s'),
290
+ 'Returning 498 for %(meth)s to %(path)s. '
291
+ 'Ratelimit (Max Sleep) %(e)s',
293
292
  {'meth': req.method, 'path': path, 'e': str(e)})
294
293
  error_resp = Response(status='498 Rate Limited',
295
294
  body='Slow down', request=req)
@@ -309,7 +308,7 @@ class RateLimitMiddleware(object):
309
308
  self.memcache_client = cache_from_env(env)
310
309
  if not self.memcache_client:
311
310
  self.logger.warning(
312
- _('Warning: Cannot ratelimit without a memcached client'))
311
+ 'Warning: Cannot ratelimit without a memcached client')
313
312
  return self.app(env, start_response)
314
313
  try:
315
314
  version, account, container, obj = req.split_path(1, 4, True)
@@ -20,7 +20,6 @@ import time
20
20
  from resource import getpagesize
21
21
 
22
22
  from swift import __version__ as swiftver
23
- from swift import gettext_ as _
24
23
  from swift.common.constraints import check_mount
25
24
  from swift.common.storage_policy import POLICIES
26
25
  from swift.common.swob import Request, Response
@@ -89,11 +88,11 @@ class ReconMiddleware(object):
89
88
  if err.errno == errno.ENOENT and ignore_missing:
90
89
  pass
91
90
  else:
92
- self.logger.exception(_('Error reading recon cache file'))
91
+ self.logger.exception('Error reading recon cache file')
93
92
  except ValueError:
94
- self.logger.exception(_('Error parsing recon cache file'))
93
+ self.logger.exception('Error parsing recon cache file')
95
94
  except Exception:
96
- self.logger.exception(_('Error retrieving recon data'))
95
+ self.logger.exception('Error retrieving recon data')
97
96
  return dict((key, None) for key in cache_keys)
98
97
 
99
98
  def get_version(self):
@@ -181,7 +180,7 @@ class ReconMiddleware(object):
181
180
  try:
182
181
  return {self.devices: os.listdir(self.devices)}
183
182
  except Exception:
184
- self.logger.exception(_('Error listing devices'))
183
+ self.logger.exception('Error listing devices')
185
184
  return {self.devices: None}
186
185
 
187
186
  def get_updater_info(self, recon_type):
@@ -277,7 +276,7 @@ class ReconMiddleware(object):
277
276
  except IOError as err:
278
277
  sums[ringfile] = None
279
278
  if err.errno != errno.ENOENT:
280
- self.logger.exception(_('Error reading ringfile'))
279
+ self.logger.exception('Error reading ringfile')
281
280
  return sums
282
281
 
283
282
  def get_swift_conf_md5(self):
@@ -287,7 +286,7 @@ class ReconMiddleware(object):
287
286
  hexsum = md5_hash_for_file(SWIFT_CONF_FILE)
288
287
  except IOError as err:
289
288
  if err.errno != errno.ENOENT:
290
- self.logger.exception(_('Error reading swift.conf'))
289
+ self.logger.exception('Error reading swift.conf')
291
290
  return {SWIFT_CONF_FILE: hexsum}
292
291
 
293
292
  def get_quarantine_count(self):
@@ -133,6 +133,15 @@ class BaseAclHandler(object):
133
133
  query = {}
134
134
  else:
135
135
  query = {'version-id': version_id}
136
+ if self.req.method == 'HEAD':
137
+ # This HEAD for ACL is going to also be the definitive response
138
+ # to the client so we need to include client params. We don't
139
+ # do this for other client request methods because they may
140
+ # have invalid combinations of params and headers for a swift
141
+ # HEAD request.
142
+ part_number = self.req.params.get('partNumber')
143
+ if part_number is not None:
144
+ query['part-number'] = part_number
136
145
  resp = self.req.get_acl_response(app, 'HEAD',
137
146
  container, obj,
138
147
  headers, query=query)
@@ -168,7 +177,7 @@ class BaseAclHandler(object):
168
177
  elem = fromstring(body, ACL.root_tag)
169
178
  acl = ACL.from_elem(
170
179
  elem, True, self.req.conf.allow_no_owner)
171
- except(XMLSyntaxError, DocumentInvalid):
180
+ except (XMLSyntaxError, DocumentInvalid):
172
181
  raise MalformedACLError()
173
182
  except Exception as e:
174
183
  self.logger.error(e)
@@ -33,6 +33,8 @@ from swift.common.middleware.s3api.controllers.versioning import \
33
33
  VersioningController
34
34
  from swift.common.middleware.s3api.controllers.tagging import \
35
35
  TaggingController
36
+ from swift.common.middleware.s3api.controllers.object_lock import \
37
+ ObjectLockController
36
38
 
37
39
  __all__ = [
38
40
  'Controller',
@@ -50,6 +52,7 @@ __all__ = [
50
52
  'LoggingStatusController',
51
53
  'VersioningController',
52
54
  'TaggingController',
55
+ 'ObjectLockController',
53
56
 
54
57
  'UnsupportedController',
55
58
  ]
@@ -19,8 +19,9 @@ from swift.common.utils import public
19
19
 
20
20
  from swift.common.middleware.s3api.exception import ACLError
21
21
  from swift.common.middleware.s3api.controllers.base import Controller
22
- from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented,\
23
- MalformedACLError, UnexpectedContent, MissingSecurityHeader
22
+ from swift.common.middleware.s3api.s3response import (
23
+ HTTPOk, S3NotImplemented, MalformedACLError, UnexpectedContent,
24
+ MissingSecurityHeader)
24
25
  from swift.common.middleware.s3api.etree import Element, SubElement, tostring
25
26
  from swift.common.middleware.s3api.acl_utils import swift_acl_translate, \
26
27
  XMLNS_XSI
@@ -18,8 +18,8 @@ from swift.common.utils import public
18
18
  from swift.common.middleware.s3api.controllers.base import Controller, \
19
19
  bucket_operation
20
20
  from swift.common.middleware.s3api.etree import Element, tostring
21
- from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented,\
22
- NoLoggingStatusForKey
21
+ from swift.common.middleware.s3api.s3response import (
22
+ HTTPOk, S3NotImplemented, NoLoggingStatusForKey)
23
23
 
24
24
 
25
25
  class LoggingStatusController(Controller):
@@ -68,8 +68,9 @@ import time
68
68
  import six
69
69
 
70
70
  from swift.common import constraints
71
- from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, wsgi_to_str
72
- from swift.common.utils import json, public, reiterate, md5
71
+ from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, \
72
+ wsgi_to_str
73
+ from swift.common.utils import json, public, reiterate, md5, Timestamp
73
74
  from swift.common.db import utf8encode
74
75
  from swift.common.request_helpers import get_container_update_override_key, \
75
76
  get_param
@@ -82,7 +83,7 @@ from swift.common.middleware.s3api.s3response import InvalidArgument, \
82
83
  ErrorResponse, MalformedXML, BadDigest, KeyTooLongError, \
83
84
  InvalidPart, BucketAlreadyExists, EntityTooSmall, InvalidPartOrder, \
84
85
  InvalidRequest, HTTPOk, HTTPNoContent, NoSuchKey, NoSuchUpload, \
85
- NoSuchBucket, BucketAlreadyOwnedByYou
86
+ NoSuchBucket, BucketAlreadyOwnedByYou, ServiceUnavailable
86
87
  from swift.common.middleware.s3api.utils import unique_id, \
87
88
  MULTIUPLOAD_SUFFIX, S3Timestamp, sysmeta_header
88
89
  from swift.common.middleware.s3api.etree import Element, SubElement, \
@@ -96,6 +97,17 @@ MAX_COMPLETE_UPLOAD_BODY_SIZE = 2048 * 1024
96
97
 
97
98
 
98
99
  def _get_upload_info(req, app, upload_id):
100
+ """
101
+ Make a HEAD request for existing upload object metadata. Tries the upload
102
+ marker first, and then falls back to the manifest object.
103
+
104
+ :param req: an S3Request object.
105
+ :param app: the wsgi app.
106
+ :param upload_id: the upload id.
107
+ :returns: a tuple of (S3Response, boolean) where the boolean is True if the
108
+ response is from the upload marker and False otherwise.
109
+ :raises: NoSuchUpload if neither the marker nor the manifest were found.
110
+ """
99
111
 
100
112
  container = req.container_name + MULTIUPLOAD_SUFFIX
101
113
  obj = '%s/%s' % (req.object_name, upload_id)
@@ -106,14 +118,17 @@ def _get_upload_info(req, app, upload_id):
106
118
  # it off for now...
107
119
  copy_source = req.headers.pop('X-Amz-Copy-Source', None)
108
120
  try:
109
- return req.get_response(app, 'HEAD', container=container, obj=obj)
121
+ resp = req.get_response(app, 'HEAD', container=container, obj=obj)
122
+ return resp, True
110
123
  except NoSuchKey:
124
+ # ensure consistent path and policy are logged despite manifest HEAD
111
125
  upload_marker_path = req.environ.get('s3api.backend_path')
126
+ policy_index = req.policy_index
112
127
  try:
113
128
  resp = req.get_response(app, 'HEAD')
114
129
  if resp.sysmeta_headers.get(sysmeta_header(
115
130
  'object', 'upload-id')) == upload_id:
116
- return resp
131
+ return resp, False
117
132
  except NoSuchKey:
118
133
  pass
119
134
  finally:
@@ -121,6 +136,8 @@ def _get_upload_info(req, app, upload_id):
121
136
  # path, so put it back
122
137
  if upload_marker_path is not None:
123
138
  req.environ['s3api.backend_path'] = upload_marker_path
139
+ if policy_index is not None:
140
+ req.policy_index = policy_index
124
141
  raise NoSuchUpload(upload_id=upload_id)
125
142
  finally:
126
143
  # ...making sure to restore any copy-source before returning
@@ -179,15 +196,7 @@ class PartController(Controller):
179
196
  raise InvalidArgument('ResourceType', 'partNumber',
180
197
  'Unexpected query string parameter')
181
198
 
182
- try:
183
- part_number = int(get_param(req, 'partNumber'))
184
- if part_number < 1 or self.conf.max_upload_part_num < part_number:
185
- raise Exception()
186
- except Exception:
187
- err_msg = 'Part number must be an integer between 1 and %d,' \
188
- ' inclusive' % self.conf.max_upload_part_num
189
- raise InvalidArgument('partNumber', get_param(req, 'partNumber'),
190
- err_msg)
199
+ part_number = req.validate_part_number()
191
200
 
192
201
  upload_id = get_param(req, 'uploadId')
193
202
  _get_upload_info(req, self.app, upload_id)
@@ -651,7 +660,14 @@ class UploadController(Controller):
651
660
  Handles Complete Multipart Upload.
652
661
  """
653
662
  upload_id = get_param(req, 'uploadId')
654
- resp = _get_upload_info(req, self.app, upload_id)
663
+ resp, is_marker = _get_upload_info(req, self.app, upload_id)
664
+ if (is_marker and
665
+ resp.sw_headers.get('X-Backend-Timestamp') >= Timestamp.now()):
666
+ # Somehow the marker was created in the future w.r.t. this thread's
667
+ # clock. The manifest PUT may succeed but the subsequent marker
668
+ # DELETE will fail, so don't attempt either.
669
+ raise ServiceUnavailable
670
+
655
671
  headers = {'Accept': 'application/json',
656
672
  sysmeta_header('object', 'upload-id'): upload_id}
657
673
  for key, val in resp.headers.items():
@@ -90,8 +90,14 @@ class ObjectController(Controller):
90
90
  if version_id not in ('null', None) and \
91
91
  'object_versioning' not in get_swift_info():
92
92
  raise S3NotImplemented()
93
+ part_number = req.validate_part_number(check_max=False)
94
+
95
+ query = {}
96
+ if version_id is not None:
97
+ query['version-id'] = version_id
98
+ if part_number is not None:
99
+ query['part-number'] = part_number
93
100
 
94
- query = {} if version_id is None else {'version-id': version_id}
95
101
  if version_id not in ('null', None):
96
102
  container_info = req.get_container_info(self.app)
97
103
  if not container_info.get(
@@ -101,6 +107,19 @@ class ObjectController(Controller):
101
107
 
102
108
  resp = req.get_response(self.app, query=query)
103
109
 
110
+ if not resp.is_slo:
111
+ # SLO ignores part_number for non-slo objects, but s3api only
112
+ # allows the query param for non-MPU if it's exactly 1.
113
+ part_number = req.validate_part_number(parts_count=1)
114
+ if part_number == 1:
115
+ # When the query param *is* exactly 1 the response status code
116
+ # and headers are updated.
117
+ resp.status = HTTP_PARTIAL_CONTENT
118
+ resp.headers['Content-Range'] = \
119
+ 'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1,
120
+ resp.headers['Content-Length'])
121
+ # else: part_number is None
122
+
104
123
  if req.method == 'HEAD':
105
124
  resp.app_iter = None
106
125
 
@@ -0,0 +1,44 @@
1
+ # Copyright (c) 2010-2023 OpenStack Foundation
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
+ # implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from swift.common.utils import public
17
+
18
+ from swift.common.middleware.s3api.controllers.base import Controller, \
19
+ bucket_operation, S3NotImplemented
20
+ from swift.common.middleware.s3api.s3response import \
21
+ ObjectLockConfigurationNotFoundError
22
+
23
+
24
+ class ObjectLockController(Controller):
25
+ """
26
+ Handles GET object-lock request, which always returns
27
+ <ObjectLockEnabled>Disabled</ObjectLockEnabled>
28
+ """
29
+ @public
30
+ @bucket_operation
31
+ def GET(self, req):
32
+ """
33
+ Handles GET object-lock param calls.
34
+ """
35
+ raise ObjectLockConfigurationNotFoundError(req.container_name)
36
+
37
+ @public
38
+ @bucket_operation
39
+ def PUT(self, req):
40
+ """
41
+ Handles PUT object-lock param calls.
42
+ """
43
+ # Basically we don't support it, so return a 501
44
+ raise S3NotImplemented('The requested resource is not implemented')
@@ -149,6 +149,7 @@ from six.moves.urllib.parse import parse_qs
149
149
  from swift.common.constraints import valid_api_version
150
150
  from swift.common.middleware.listing_formats import \
151
151
  MAX_CONTAINER_LISTING_CONTENT_LENGTH
152
+ from swift.common.request_helpers import append_log_info
152
153
  from swift.common.wsgi import PipelineWrapper, loadcontext, WSGIContext
153
154
 
154
155
  from swift.common.middleware import app_property
@@ -353,6 +354,7 @@ class S3ApiMiddleware(object):
353
354
  self.logger.debug(e.cause)
354
355
  except ErrorResponse as err_resp:
355
356
  self.logger.increment(err_resp.metric_name)
357
+ append_log_info(env, 's3:err:%s' % err_resp.summary)
356
358
  if isinstance(err_resp, InternalError):
357
359
  self.logger.exception(err_resp)
358
360
  resp = err_resp
@@ -366,6 +368,7 @@ class S3ApiMiddleware(object):
366
368
 
367
369
  if 's3api.backend_path' in env and 'swift.backend_path' not in env:
368
370
  env['swift.backend_path'] = env['s3api.backend_path']
371
+
369
372
  return resp(env, start_response)
370
373
 
371
374
  def handle_request(self, req):
@@ -390,6 +393,9 @@ class S3ApiMiddleware(object):
390
393
  raise MethodNotAllowed(req.method,
391
394
  req.controller.resource_type())
392
395
 
396
+ if req.policy_index is not None:
397
+ res.headers.setdefault('X-Backend-Storage-Policy-Index',
398
+ req.policy_index)
393
399
  return res
394
400
 
395
401
  def check_pipeline(self, wsgi_conf):