swift 2.31.1__py2.py3-none-any.whl → 2.32.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. swift/cli/info.py +9 -2
  2. swift/cli/ringbuilder.py +5 -1
  3. swift/common/container_sync_realms.py +6 -7
  4. swift/common/daemon.py +7 -3
  5. swift/common/db.py +22 -7
  6. swift/common/db_replicator.py +19 -20
  7. swift/common/direct_client.py +63 -14
  8. swift/common/internal_client.py +24 -3
  9. swift/common/manager.py +43 -44
  10. swift/common/memcached.py +168 -74
  11. swift/common/middleware/__init__.py +4 -0
  12. swift/common/middleware/account_quotas.py +98 -40
  13. swift/common/middleware/backend_ratelimit.py +6 -4
  14. swift/common/middleware/crossdomain.py +21 -8
  15. swift/common/middleware/listing_formats.py +26 -38
  16. swift/common/middleware/proxy_logging.py +12 -9
  17. swift/common/middleware/s3api/controllers/bucket.py +8 -2
  18. swift/common/middleware/s3api/s3api.py +9 -4
  19. swift/common/middleware/s3api/s3request.py +32 -24
  20. swift/common/middleware/s3api/s3response.py +10 -1
  21. swift/common/middleware/tempauth.py +9 -10
  22. swift/common/middleware/versioned_writes/__init__.py +0 -3
  23. swift/common/middleware/versioned_writes/object_versioning.py +22 -5
  24. swift/common/middleware/x_profile/html_viewer.py +1 -1
  25. swift/common/middleware/xprofile.py +5 -0
  26. swift/common/request_helpers.py +1 -2
  27. swift/common/ring/ring.py +22 -19
  28. swift/common/swob.py +2 -1
  29. swift/common/{utils.py → utils/__init__.py} +610 -1146
  30. swift/common/utils/ipaddrs.py +256 -0
  31. swift/common/utils/libc.py +345 -0
  32. swift/common/utils/timestamp.py +399 -0
  33. swift/common/wsgi.py +70 -39
  34. swift/container/backend.py +106 -38
  35. swift/container/server.py +11 -2
  36. swift/container/sharder.py +34 -15
  37. swift/locale/de/LC_MESSAGES/swift.po +1 -320
  38. swift/locale/en_GB/LC_MESSAGES/swift.po +1 -347
  39. swift/locale/es/LC_MESSAGES/swift.po +1 -279
  40. swift/locale/fr/LC_MESSAGES/swift.po +1 -209
  41. swift/locale/it/LC_MESSAGES/swift.po +1 -207
  42. swift/locale/ja/LC_MESSAGES/swift.po +2 -278
  43. swift/locale/ko_KR/LC_MESSAGES/swift.po +3 -303
  44. swift/locale/pt_BR/LC_MESSAGES/swift.po +1 -204
  45. swift/locale/ru/LC_MESSAGES/swift.po +1 -203
  46. swift/locale/tr_TR/LC_MESSAGES/swift.po +1 -192
  47. swift/locale/zh_CN/LC_MESSAGES/swift.po +1 -192
  48. swift/locale/zh_TW/LC_MESSAGES/swift.po +1 -193
  49. swift/obj/diskfile.py +19 -6
  50. swift/obj/server.py +20 -6
  51. swift/obj/ssync_receiver.py +19 -9
  52. swift/obj/ssync_sender.py +10 -10
  53. swift/proxy/controllers/account.py +7 -7
  54. swift/proxy/controllers/base.py +374 -366
  55. swift/proxy/controllers/container.py +112 -53
  56. swift/proxy/controllers/obj.py +254 -390
  57. swift/proxy/server.py +3 -8
  58. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-server +1 -1
  59. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-server +1 -1
  60. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-drive-audit +45 -14
  61. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-server +1 -1
  62. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-proxy-server +1 -1
  63. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/AUTHORS +4 -0
  64. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/METADATA +32 -35
  65. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/RECORD +103 -100
  66. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/WHEEL +1 -1
  67. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/entry_points.txt +0 -1
  68. swift-2.32.1.dist-info/pbr.json +1 -0
  69. swift-2.31.1.dist-info/pbr.json +0 -1
  70. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-audit +0 -0
  71. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-auditor +0 -0
  72. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-info +0 -0
  73. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-reaper +0 -0
  74. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-replicator +0 -0
  75. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-config +0 -0
  76. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-auditor +0 -0
  77. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-info +0 -0
  78. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-reconciler +0 -0
  79. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-replicator +0 -0
  80. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-sharder +0 -0
  81. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-sync +0 -0
  82. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-updater +0 -0
  83. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-dispersion-populate +0 -0
  84. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-dispersion-report +0 -0
  85. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-form-signature +0 -0
  86. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-get-nodes +0 -0
  87. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-init +0 -0
  88. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-auditor +0 -0
  89. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-expirer +0 -0
  90. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-info +0 -0
  91. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-reconstructor +0 -0
  92. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-relinker +0 -0
  93. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-replicator +0 -0
  94. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-updater +0 -0
  95. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-oldies +0 -0
  96. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-orphans +0 -0
  97. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-recon +0 -0
  98. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-recon-cron +0 -0
  99. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-reconciler-enqueue +0 -0
  100. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-builder +0 -0
  101. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-builder-analyzer +0 -0
  102. {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-composer +0 -0
  103. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/LICENSE +0 -0
  104. {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/top_level.txt +0 -0
@@ -23,20 +23,24 @@ class CrossDomainMiddleware(object):
23
23
  Cross domain middleware used to respond to requests for cross domain
24
24
  policy information.
25
25
 
26
- If the path is /crossdomain.xml it will respond with an xml cross domain
27
- policy document. This allows web pages hosted elsewhere to use client
28
- side technologies such as Flash, Java and Silverlight to interact
26
+ If the path is ``/crossdomain.xml`` it will respond with an xml cross
27
+ domain policy document. This allows web pages hosted elsewhere to use
28
+ client side technologies such as Flash, Java and Silverlight to interact
29
29
  with the Swift API.
30
30
 
31
31
  To enable this middleware, add it to the pipeline in your proxy-server.conf
32
32
  file. It should be added before any authentication (e.g., tempauth or
33
33
  keystone) middleware. In this example ellipsis (...) indicate other
34
- middleware you may have chosen to use::
34
+ middleware you may have chosen to use:
35
+
36
+ .. code:: cfg
35
37
 
36
38
  [pipeline:main]
37
39
  pipeline = ... crossdomain ... authtoken ... proxy-server
38
40
 
39
- And add a filter section, such as::
41
+ And add a filter section, such as:
42
+
43
+ .. code:: cfg
40
44
 
41
45
  [filter:crossdomain]
42
46
  use = egg:swift#crossdomain
@@ -45,13 +49,22 @@ class CrossDomainMiddleware(object):
45
49
 
46
50
  For continuation lines, put some whitespace before the continuation
47
51
  text. Ensure you put a completely blank line to terminate the
48
- cross_domain_policy value.
52
+ ``cross_domain_policy`` value.
53
+
54
+ The ``cross_domain_policy`` name/value is optional. If omitted, the policy
55
+ defaults as if you had specified:
49
56
 
50
- The cross_domain_policy name/value is optional. If omitted, the policy
51
- defaults as if you had specified::
57
+ .. code:: cfg
52
58
 
53
59
  cross_domain_policy = <allow-access-from domain="*" secure="false" />
54
60
 
61
+ .. note::
62
+
63
+ The default policy is very permissive; this is appropriate
64
+ for most public cloud deployments, but may not be appropriate
65
+ for all deployments. See also:
66
+ `CWE-942 <https://cwe.mitre.org/data/definitions/942.html>`__
67
+
55
68
 
56
69
  """
57
70
 
@@ -18,6 +18,7 @@ import six
18
18
  from xml.etree.cElementTree import Element, SubElement, tostring
19
19
 
20
20
  from swift.common.constraints import valid_api_version
21
+ from swift.common.header_key_dict import HeaderKeyDict
21
22
  from swift.common.http import HTTP_NO_CONTENT
22
23
  from swift.common.request_helpers import get_param
23
24
  from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
@@ -178,52 +179,39 @@ class ListingFilter(object):
178
179
  start_response(status, headers)
179
180
  return resp_iter
180
181
 
181
- header_to_index = {}
182
- resp_content_type = resp_length = None
183
- for i, (header, value) in enumerate(headers):
184
- header = header.lower()
185
- if header == 'content-type':
186
- header_to_index[header] = i
187
- resp_content_type = value.partition(';')[0]
188
- elif header == 'content-length':
189
- header_to_index[header] = i
190
- resp_length = int(value)
191
- elif header == 'vary':
192
- header_to_index[header] = i
193
-
194
182
  if not status.startswith(('200 ', '204 ')):
195
183
  start_response(status, headers)
196
184
  return resp_iter
197
185
 
186
+ headers_dict = HeaderKeyDict(headers)
187
+ resp_content_type = headers_dict.get(
188
+ 'content-type', '').partition(';')[0]
189
+ resp_length = headers_dict.get('content-length')
190
+
198
191
  if can_vary:
199
- if 'vary' in header_to_index:
200
- value = headers[header_to_index['vary']][1]
192
+ if 'vary' in headers_dict:
193
+ value = headers_dict['vary']
201
194
  if 'accept' not in list_from_csv(value.lower()):
202
- headers[header_to_index['vary']] = (
203
- 'Vary', value + ', Accept')
195
+ headers_dict['vary'] = value + ', Accept'
204
196
  else:
205
- headers.append(('Vary', 'Accept'))
197
+ headers_dict['vary'] = 'Accept'
206
198
 
207
199
  if resp_content_type != 'application/json':
208
- start_response(status, headers)
200
+ start_response(status, list(headers_dict.items()))
209
201
  return resp_iter
210
202
 
211
- if resp_length is None or \
212
- resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
213
- start_response(status, headers)
203
+ if req.method == 'HEAD':
204
+ headers_dict['content-type'] = out_content_type + '; charset=utf-8'
205
+ # proxy logging (and maybe other mw?) seem to be good about
206
+ # sticking this on HEAD/204 but we do it here to be responsible
207
+ # and explicit
208
+ headers_dict['content-length'] = 0
209
+ start_response(status, list(headers_dict.items()))
214
210
  return resp_iter
215
211
 
216
- def set_header(header, value):
217
- if value is None:
218
- del headers[header_to_index[header]]
219
- else:
220
- headers[header_to_index[header]] = (
221
- headers[header_to_index[header]][0], str(value))
222
-
223
- if req.method == 'HEAD':
224
- set_header('content-type', out_content_type + '; charset=utf-8')
225
- set_header('content-length', None) # don't know, can't determine
226
- start_response(status, headers)
212
+ if resp_length is None or \
213
+ int(resp_length) > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
214
+ start_response(status, list(headers_dict.items()))
227
215
  return resp_iter
228
216
 
229
217
  body = b''.join(resp_iter)
@@ -237,7 +225,7 @@ class ListingFilter(object):
237
225
  except ValueError:
238
226
  # Static web listing that's returning invalid JSON?
239
227
  # Just pass it straight through; that's about all we *can* do.
240
- start_response(status, headers)
228
+ start_response(status, list(headers_dict.items()))
241
229
  return [body]
242
230
 
243
231
  if not req.allow_reserved_names:
@@ -257,16 +245,16 @@ class ListingFilter(object):
257
245
  body = json.dumps(listing).encode('ascii')
258
246
  except KeyError:
259
247
  # listing was in a bad format -- funky static web listing??
260
- start_response(status, headers)
248
+ start_response(status, list(headers_dict.items()))
261
249
  return [body]
262
250
 
263
251
  if not body:
264
252
  status = '%s %s' % (HTTP_NO_CONTENT,
265
253
  RESPONSE_REASONS[HTTP_NO_CONTENT][0])
266
254
 
267
- set_header('content-type', out_content_type + '; charset=utf-8')
268
- set_header('content-length', len(body))
269
- start_response(status, headers)
255
+ headers_dict['content-type'] = out_content_type + '; charset=utf-8'
256
+ headers_dict['content-length'] = len(body)
257
+ start_response(status, list(headers_dict.items()))
270
258
  return [body]
271
259
 
272
260
 
@@ -19,12 +19,12 @@ Logging middleware for the Swift proxy.
19
19
  This serves as both the default logging implementation and an example of how
20
20
  to plug in your own logging format/method.
21
21
 
22
- The logging format implemented below is as follows:
22
+ The logging format implemented below is as follows::
23
23
 
24
- client_ip remote_addr end_time.datetime method path protocol
25
- status_int referer user_agent auth_token bytes_recvd bytes_sent
26
- client_etag transaction_id headers request_time source log_info
27
- start_time end_time policy_index
24
+ client_ip remote_addr end_time.datetime method path protocol
25
+ status_int referer user_agent auth_token bytes_recvd bytes_sent
26
+ client_etag transaction_id headers request_time source log_info
27
+ start_time end_time policy_index
28
28
 
29
29
  These values are space-separated, and each is url-encoded, so that they can
30
30
  be separated with a simple .split()
@@ -49,6 +49,11 @@ be separated with a simple .split()
49
49
  * Values that are missing (e.g. due to a header not being present) or zero
50
50
  are generally represented by a single hyphen ('-').
51
51
 
52
+ .. note::
53
+ The message format may be configured using the ``log_msg_template`` option,
54
+ allowing fields to be added, removed, re-ordered, and even anonymized. For
55
+ more information, see https://docs.openstack.org/swift/latest/logs.html
56
+
52
57
  The proxy-logging can be used twice in the proxy server's pipeline when there
53
58
  is middleware installed that can return custom responses that don't follow the
54
59
  standard pipeline to the proxy server.
@@ -78,7 +83,7 @@ from swift.common.middleware.catch_errors import enforce_byte_count
78
83
  from swift.common.swob import Request
79
84
  from swift.common.utils import (get_logger, get_remote_client,
80
85
  config_true_value, reiterate,
81
- close_if_possible,
86
+ close_if_possible, cap_length,
82
87
  InputProxy, list_from_csv, get_policy_index,
83
88
  split_path, StrAnonymizer, StrFormatTime,
84
89
  LogStringFormatter)
@@ -200,9 +205,7 @@ class ProxyLoggingMiddleware(object):
200
205
  env['swift.proxy_access_log_made'] = True
201
206
 
202
207
  def obscure_sensitive(self, value):
203
- if value and len(value) > self.reveal_sensitive_prefix:
204
- return value[:self.reveal_sensitive_prefix] + '...'
205
- return value
208
+ return cap_length(value, self.reveal_sensitive_prefix)
206
209
 
207
210
  def obscure_req(self, req):
208
211
  for header in get_sensitive_headers():
@@ -23,7 +23,8 @@ from swift.common import swob
23
23
  from swift.common.http import HTTP_OK
24
24
  from swift.common.middleware.versioned_writes.object_versioning import \
25
25
  DELETE_MARKER_CONTENT_TYPE
26
- from swift.common.utils import json, public, config_true_value, Timestamp
26
+ from swift.common.utils import json, public, config_true_value, Timestamp, \
27
+ cap_length
27
28
  from swift.common.registry import get_swift_info
28
29
 
29
30
  from swift.common.middleware.s3api.controllers.base import Controller
@@ -343,7 +344,12 @@ class BucketController(Controller):
343
344
 
344
345
  resp = req.get_response(self.app, query=query)
345
346
 
346
- objects = json.loads(resp.body)
347
+ try:
348
+ objects = json.loads(resp.body)
349
+ except (TypeError, ValueError):
350
+ self.logger.error('Got non-JSON response trying to list %s: %r',
351
+ req.path, cap_length(resp.body, 60))
352
+ raise
347
353
 
348
354
  is_truncated = max_keys > 0 and len(objects) > max_keys
349
355
  objects = objects[:max_keys]
@@ -151,6 +151,7 @@ from swift.common.middleware.listing_formats import \
151
151
  MAX_CONTAINER_LISTING_CONTENT_LENGTH
152
152
  from swift.common.wsgi import PipelineWrapper, loadcontext, WSGIContext
153
153
 
154
+ from swift.common.middleware import app_property
154
155
  from swift.common.middleware.s3api.exception import NotS3Request, \
155
156
  InvalidSubresource
156
157
  from swift.common.middleware.s3api.s3request import get_request_class
@@ -167,9 +168,12 @@ from swift.common.registry import register_swift_info, \
167
168
  class ListingEtagMiddleware(object):
168
169
  def __init__(self, app):
169
170
  self.app = app
170
- # Pass this along so get_container_info will have the configured
171
- # odds to skip cache
172
- self._pipeline_final_app = app._pipeline_final_app
171
+
172
+ # Pass these along so get_container_info will have the configured
173
+ # odds to skip cache
174
+ _pipeline_final_app = app_property('_pipeline_final_app')
175
+ _pipeline_request_logging_app = app_property(
176
+ '_pipeline_request_logging_app')
173
177
 
174
178
  def __call__(self, env, start_response):
175
179
  # a lot of this is cribbed from listing_formats / swob.Request
@@ -293,7 +297,7 @@ class S3ApiMiddleware(object):
293
297
  wsgi_conf.get('ratelimit_as_client_error', False))
294
298
 
295
299
  self.logger = get_logger(
296
- wsgi_conf, log_route=wsgi_conf.get('log_name', 's3api'))
300
+ wsgi_conf, log_route='s3api', statsd_tail_prefix='s3api')
297
301
  self.check_pipeline(wsgi_conf)
298
302
 
299
303
  def is_s3_cors_preflight(self, env):
@@ -348,6 +352,7 @@ class S3ApiMiddleware(object):
348
352
  except InvalidSubresource as e:
349
353
  self.logger.debug(e.cause)
350
354
  except ErrorResponse as err_resp:
355
+ self.logger.increment(err_resp.metric_name)
351
356
  if isinstance(err_resp, InternalError):
352
357
  self.logger.exception(err_resp)
353
358
  resp = err_resp
@@ -189,11 +189,13 @@ class SigV4Mixin(object):
189
189
  timestamp = mktime(self.headers.get('Date'))
190
190
  except (ValueError, TypeError):
191
191
  raise AccessDenied('AWS authentication requires a valid Date '
192
- 'or x-amz-date header')
192
+ 'or x-amz-date header',
193
+ reason='invalid_date')
193
194
 
194
195
  if timestamp < 0:
195
196
  raise AccessDenied('AWS authentication requires a valid Date '
196
- 'or x-amz-date header')
197
+ 'or x-amz-date header',
198
+ reason='invalid_date')
197
199
 
198
200
  try:
199
201
  self._timestamp = S3Timestamp(timestamp)
@@ -214,7 +216,7 @@ class SigV4Mixin(object):
214
216
  try:
215
217
  expires = int(self.params['X-Amz-Expires'])
216
218
  except KeyError:
217
- raise AccessDenied()
219
+ raise AccessDenied(reason='invalid_expires')
218
220
  except ValueError:
219
221
  err = 'X-Amz-Expires should be a number'
220
222
  else:
@@ -230,14 +232,14 @@ class SigV4Mixin(object):
230
232
  raise AuthorizationQueryParametersError(err)
231
233
 
232
234
  if int(self.timestamp) + expires < S3Timestamp.now():
233
- raise AccessDenied('Request has expired')
235
+ raise AccessDenied('Request has expired', reason='expired')
234
236
 
235
237
  def _parse_credential(self, credential_string):
236
238
  parts = credential_string.split("/")
237
239
  # credential must be in following format:
238
240
  # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request
239
241
  if not parts[0] or len(parts) != 5:
240
- raise AccessDenied()
242
+ raise AccessDenied(reason='invalid_credential')
241
243
  return dict(zip(['access', 'date', 'region', 'service', 'terminal'],
242
244
  parts))
243
245
 
@@ -257,9 +259,9 @@ class SigV4Mixin(object):
257
259
  swob.wsgi_to_str(self.params['X-Amz-Credential']))
258
260
  sig = swob.wsgi_to_str(self.params['X-Amz-Signature'])
259
261
  if not sig:
260
- raise AccessDenied()
262
+ raise AccessDenied(reason='invalid_query_auth')
261
263
  except KeyError:
262
- raise AccessDenied()
264
+ raise AccessDenied(reason='invalid_query_auth')
263
265
 
264
266
  try:
265
267
  signed_headers = swob.wsgi_to_str(
@@ -311,7 +313,7 @@ class SigV4Mixin(object):
311
313
  "Credential=")[2].split(',')[0])
312
314
  sig = auth_str.partition("Signature=")[2].split(',')[0]
313
315
  if not sig:
314
- raise AccessDenied()
316
+ raise AccessDenied(reason='invalid_header_auth')
315
317
  signed_headers = auth_str.partition(
316
318
  "SignedHeaders=")[2].split(',', 1)[0]
317
319
  if not signed_headers:
@@ -582,11 +584,13 @@ class S3Request(swob.Request):
582
584
  self.headers.get('Date')))
583
585
  except ValueError:
584
586
  raise AccessDenied('AWS authentication requires a valid Date '
585
- 'or x-amz-date header')
587
+ 'or x-amz-date header',
588
+ reason='invalid_date')
586
589
 
587
590
  if timestamp < 0:
588
591
  raise AccessDenied('AWS authentication requires a valid Date '
589
- 'or x-amz-date header')
592
+ 'or x-amz-date header',
593
+ reason='invalid_date')
590
594
  try:
591
595
  self._timestamp = S3Timestamp(timestamp)
592
596
  except ValueError:
@@ -658,10 +662,10 @@ class S3Request(swob.Request):
658
662
  expires = swob.wsgi_to_str(self.params['Expires'])
659
663
  sig = swob.wsgi_to_str(self.params['Signature'])
660
664
  except KeyError:
661
- raise AccessDenied()
665
+ raise AccessDenied(reason='invalid_query_auth')
662
666
 
663
667
  if not all([access, sig, expires]):
664
- raise AccessDenied()
668
+ raise AccessDenied(reason='invalid_query_auth')
665
669
 
666
670
  return access, sig
667
671
 
@@ -674,7 +678,7 @@ class S3Request(swob.Request):
674
678
  """
675
679
  auth_str = swob.wsgi_to_str(self.headers['Authorization'])
676
680
  if not auth_str.startswith('AWS ') or ':' not in auth_str:
677
- raise AccessDenied()
681
+ raise AccessDenied(reason='invalid_header_auth')
678
682
  # This means signature format V2
679
683
  access, sig = auth_str.split(' ', 1)[1].rsplit(':', 1)
680
684
  return access, sig
@@ -705,15 +709,15 @@ class S3Request(swob.Request):
705
709
  try:
706
710
  ex = S3Timestamp(float(self.params['Expires']))
707
711
  except (KeyError, ValueError):
708
- raise AccessDenied()
712
+ raise AccessDenied(reason='invalid_expires')
709
713
 
710
714
  if S3Timestamp.now() > ex:
711
- raise AccessDenied('Request has expired')
715
+ raise AccessDenied('Request has expired', reason='expired')
712
716
 
713
717
  if ex >= 2 ** 31:
714
718
  raise AccessDenied(
715
719
  'Invalid date (should be seconds since epoch): %s' %
716
- self.params['Expires'])
720
+ self.params['Expires'], reason='invalid_expires')
717
721
 
718
722
  def _validate_dates(self):
719
723
  """
@@ -725,12 +729,13 @@ class S3Request(swob.Request):
725
729
  amz_date_header = self.headers.get('X-Amz-Date')
726
730
  if not date_header and not amz_date_header:
727
731
  raise AccessDenied('AWS authentication requires a valid Date '
728
- 'or x-amz-date header')
732
+ 'or x-amz-date header',
733
+ reason='invalid_date')
729
734
 
730
735
  # Anyways, request timestamp should be validated
731
736
  epoch = S3Timestamp(0)
732
737
  if self.timestamp < epoch:
733
- raise AccessDenied()
738
+ raise AccessDenied(reason='invalid_date')
734
739
 
735
740
  # If the standard date is too far ahead or behind, it is an
736
741
  # error
@@ -984,7 +989,7 @@ class S3Request(swob.Request):
984
989
  else:
985
990
  # Should have already raised NotS3Request in _parse_auth_info,
986
991
  # but as a sanity check...
987
- raise AccessDenied()
992
+ raise AccessDenied(reason='not_s3')
988
993
 
989
994
  for key, value in sorted(amz_headers.items()):
990
995
  buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
@@ -1348,15 +1353,18 @@ class S3Request(swob.Request):
1348
1353
  try:
1349
1354
  sw_resp = sw_req.get_response(app)
1350
1355
  except swob.HTTPException as err:
1356
+ # Maybe a 422 from HashingInput? Put something in
1357
+ # s3api.backend_path - hopefully by now any modifications to the
1358
+ # path (e.g. tenant to account translation) will have been made by
1359
+ # auth middleware
1360
+ self.environ['s3api.backend_path'] = sw_req.environ['PATH_INFO']
1351
1361
  sw_resp = err
1352
1362
  else:
1353
1363
  # reuse account
1354
1364
  _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
1355
1365
  2, 3, True)
1356
- # Propagate swift.backend_path in environ for middleware
1357
- # in pipeline that need Swift PATH_INFO like ceilometermiddleware.
1358
- self.environ['s3api.backend_path'] = \
1359
- sw_resp.environ['PATH_INFO']
1366
+ # Update s3.backend_path from the response environ
1367
+ self.environ['s3api.backend_path'] = sw_resp.environ['PATH_INFO']
1360
1368
  # Propogate backend headers back into our req headers for logging
1361
1369
  for k, v in sw_req.headers.items():
1362
1370
  if k.lower().startswith('x-backend-'):
@@ -1412,7 +1420,7 @@ class S3Request(swob.Request):
1412
1420
  raise SignatureDoesNotMatch(
1413
1421
  **self.signature_does_not_match_kwargs())
1414
1422
  if status == HTTP_FORBIDDEN:
1415
- raise AccessDenied()
1423
+ raise AccessDenied(reason='forbidden')
1416
1424
  if status == HTTP_SERVICE_UNAVAILABLE:
1417
1425
  raise ServiceUnavailable()
1418
1426
  if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):
@@ -229,11 +229,12 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
229
229
  _code = ''
230
230
  xml_declaration = True
231
231
 
232
- def __init__(self, msg=None, *args, **kwargs):
232
+ def __init__(self, msg=None, reason=None, *args, **kwargs):
233
233
  if msg:
234
234
  self._msg = msg
235
235
  if not self._code:
236
236
  self._code = self.__class__.__name__
237
+ self.reason = reason
237
238
 
238
239
  self.info = kwargs.copy()
239
240
  for reserved_key in ('headers', 'body'):
@@ -247,6 +248,14 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
247
248
  **kwargs)
248
249
  self.headers = HeaderKeyDict(self.headers)
249
250
 
251
+ @property
252
+ def metric_name(self):
253
+ parts = [str(self.status_int), self._code]
254
+ if self.reason:
255
+ parts.append(self.reason)
256
+ metric = '.'.join(parts)
257
+ return metric.replace(' ', '_')
258
+
250
259
  def _body_iter(self):
251
260
  error_elem = Element('Error')
252
261
  SubElement(error_elem, 'Code').text = self._code
@@ -804,24 +804,23 @@ class TempAuth(object):
804
804
  key = req.headers.get('x-storage-pass')
805
805
  else:
806
806
  return HTTPBadRequest(request=req)
807
+ unauthed_headers = {
808
+ 'Www-Authenticate': 'Swift realm="%s"' % (account or 'unknown'),
809
+ }
807
810
  if not all((account, user, key)):
808
811
  self.logger.increment('token_denied')
809
- realm = account or 'unknown'
810
- return HTTPUnauthorized(request=req, headers={'Www-Authenticate':
811
- 'Swift realm="%s"' %
812
- realm})
812
+ return HTTPUnauthorized(request=req, headers=unauthed_headers)
813
813
  # Authenticate user
814
+ account = wsgi_to_str(account)
815
+ user = wsgi_to_str(user)
816
+ key = wsgi_to_str(key)
814
817
  account_user = account + ':' + user
815
818
  if account_user not in self.users:
816
819
  self.logger.increment('token_denied')
817
- auth = 'Swift realm="%s"' % account
818
- return HTTPUnauthorized(request=req,
819
- headers={'Www-Authenticate': auth})
820
+ return HTTPUnauthorized(request=req, headers=unauthed_headers)
820
821
  if self.users[account_user]['key'] != key:
821
822
  self.logger.increment('token_denied')
822
- auth = 'Swift realm="unknown"'
823
- return HTTPUnauthorized(request=req,
824
- headers={'Www-Authenticate': auth})
823
+ return HTTPUnauthorized(request=req, headers=unauthed_headers)
825
824
  account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
826
825
  # Get memcache client
827
826
  memcache_client = cache_from_env(req.environ)
@@ -47,8 +47,5 @@ def filter_factory(global_conf, **local_conf):
47
47
  if 'symlink' not in get_swift_info():
48
48
  raise ValueError('object versioning requires symlinks')
49
49
  app = ObjectVersioningMiddleware(app, conf)
50
- # Pass this along so get_container_info will have the configured
51
- # odds to skip cache
52
- app._pipeline_final_app = app.app._pipeline_final_app
53
50
  return VersionedWritesMiddleware(app, conf)
54
51
  return versioning_filter
@@ -158,6 +158,7 @@ from swift.common.http import is_success, is_client_error, HTTP_NOT_FOUND, \
158
158
  from swift.common.request_helpers import get_sys_meta_prefix, \
159
159
  copy_header_subset, get_reserved_name, split_reserved_name, \
160
160
  constrain_req_limit
161
+ from swift.common.middleware import app_property
161
162
  from swift.common.middleware.symlink import TGT_OBJ_SYMLINK_HDR, \
162
163
  TGT_ETAG_SYSMETA_SYMLINK_HDR, SYMLOOP_EXTEND, ALLOW_RESERVED_NAMES, \
163
164
  TGT_BYTES_SYSMETA_SYMLINK_HDR, TGT_ACCT_SYMLINK_HDR
@@ -1165,12 +1166,22 @@ class ContainerContext(ObjectVersioningContext):
1165
1166
  params['prefix'] = get_reserved_name(params['prefix'])
1166
1167
 
1167
1168
  # NB: no end_marker support (yet)
1168
- versions_req.params = {
1169
- k: params.get(k, '')
1170
- for k in ('prefix', 'marker', 'limit', 'delimiter', 'reverse')}
1171
- versions_resp = versions_req.get_response(self.app)
1169
+ if get_container_info(versions_req.environ, self.app,
1170
+ swift_source='OV')['status'] == 404:
1171
+ # we don't usually like to LBYL like this, but 404s tend to be
1172
+ # expensive (since we check all primaries and a bunch of handoffs)
1173
+ # and we expect this to be a reasonably common way to listing
1174
+ # objects since it's more complete from the user's perspective
1175
+ # (see also: s3api and that client ecosystem)
1176
+ versions_resp = None
1177
+ else:
1178
+ versions_req.params = {
1179
+ k: params.get(k, '') for k in (
1180
+ 'prefix', 'marker', 'limit', 'delimiter', 'reverse')}
1181
+ versions_resp = versions_req.get_response(self.app)
1172
1182
 
1173
- if versions_resp.status_int == HTTP_NOT_FOUND:
1183
+ if versions_resp is None \
1184
+ or versions_resp.status_int == HTTP_NOT_FOUND:
1174
1185
  subdir_listing = [{'subdir': s} for s in subdir_set]
1175
1186
  broken_listing = []
1176
1187
  for item in current_versions.values():
@@ -1379,6 +1390,12 @@ class ObjectVersioningMiddleware(object):
1379
1390
  self.conf = conf
1380
1391
  self.logger = get_logger(conf, log_route='object_versioning')
1381
1392
 
1393
+ # Pass these along so get_container_info will have the configured
1394
+ # odds to skip cache
1395
+ _pipeline_final_app = app_property('_pipeline_final_app')
1396
+ _pipeline_request_logging_app = app_property(
1397
+ '_pipeline_request_logging_app')
1398
+
1382
1399
  def account_request(self, req, api_version, account, start_response):
1383
1400
  account_ctx = AccountContext(self.app, self.logger)
1384
1401
  if req.method == 'GET':
@@ -246,7 +246,7 @@ class HTMLViewer(object):
246
246
  if multiple:
247
247
  return value
248
248
  if isinstance(value, list):
249
- return eval(value[0]) if isinstance(default, int) else value[0]
249
+ return int(value[0]) if isinstance(default, int) else value[0]
250
250
  else:
251
251
  return value
252
252
 
@@ -16,6 +16,11 @@
16
16
  """
17
17
  Profiling middleware for Swift Servers.
18
18
 
19
+ .. note::
20
+ This middleware is intended for development and testing environments only,
21
+ not production. No authentication is expected or required for the web UI,
22
+ and profiling may incur noticeable performance penalties.
23
+
19
24
  The current implementation is based on eventlet aware profiler.(For the
20
25
  future, more profilers could be added in to collect more data for analysis.)
21
26
  Profiling all incoming requests and accumulating cpu timing statistics
@@ -27,7 +27,6 @@ import time
27
27
  import six
28
28
  from swift.common.header_key_dict import HeaderKeyDict
29
29
 
30
- from swift import gettext_ as _
31
30
  from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX, \
32
31
  CONTAINER_LISTING_LIMIT
33
32
  from swift.common.storage_policy import POLICIES
@@ -202,7 +201,7 @@ def get_name_and_placement(request, minsegs=1, maxsegs=None,
202
201
  policy = POLICIES.get_by_index(policy_index)
203
202
  if not policy:
204
203
  raise HTTPServiceUnavailable(
205
- body=_("No policy with index %s") % policy_index,
204
+ body="No policy with index %s" % policy_index,
206
205
  request=request, content_type='text/plain')
207
206
  results = split_and_validate_path(request, minsegs=minsegs,
208
207
  maxsegs=maxsegs,