swift 2.31.1__py2.py3-none-any.whl → 2.32.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/cli/info.py +9 -2
- swift/cli/ringbuilder.py +5 -1
- swift/common/container_sync_realms.py +6 -7
- swift/common/daemon.py +7 -3
- swift/common/db.py +22 -7
- swift/common/db_replicator.py +19 -20
- swift/common/direct_client.py +63 -14
- swift/common/internal_client.py +24 -3
- swift/common/manager.py +43 -44
- swift/common/memcached.py +168 -74
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +98 -40
- swift/common/middleware/backend_ratelimit.py +6 -4
- swift/common/middleware/crossdomain.py +21 -8
- swift/common/middleware/listing_formats.py +26 -38
- swift/common/middleware/proxy_logging.py +12 -9
- swift/common/middleware/s3api/controllers/bucket.py +8 -2
- swift/common/middleware/s3api/s3api.py +9 -4
- swift/common/middleware/s3api/s3request.py +32 -24
- swift/common/middleware/s3api/s3response.py +10 -1
- swift/common/middleware/tempauth.py +9 -10
- swift/common/middleware/versioned_writes/__init__.py +0 -3
- swift/common/middleware/versioned_writes/object_versioning.py +22 -5
- swift/common/middleware/x_profile/html_viewer.py +1 -1
- swift/common/middleware/xprofile.py +5 -0
- swift/common/request_helpers.py +1 -2
- swift/common/ring/ring.py +22 -19
- swift/common/swob.py +2 -1
- swift/common/{utils.py → utils/__init__.py} +610 -1146
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/timestamp.py +399 -0
- swift/common/wsgi.py +70 -39
- swift/container/backend.py +106 -38
- swift/container/server.py +11 -2
- swift/container/sharder.py +34 -15
- swift/locale/de/LC_MESSAGES/swift.po +1 -320
- swift/locale/en_GB/LC_MESSAGES/swift.po +1 -347
- swift/locale/es/LC_MESSAGES/swift.po +1 -279
- swift/locale/fr/LC_MESSAGES/swift.po +1 -209
- swift/locale/it/LC_MESSAGES/swift.po +1 -207
- swift/locale/ja/LC_MESSAGES/swift.po +2 -278
- swift/locale/ko_KR/LC_MESSAGES/swift.po +3 -303
- swift/locale/pt_BR/LC_MESSAGES/swift.po +1 -204
- swift/locale/ru/LC_MESSAGES/swift.po +1 -203
- swift/locale/tr_TR/LC_MESSAGES/swift.po +1 -192
- swift/locale/zh_CN/LC_MESSAGES/swift.po +1 -192
- swift/locale/zh_TW/LC_MESSAGES/swift.po +1 -193
- swift/obj/diskfile.py +19 -6
- swift/obj/server.py +20 -6
- swift/obj/ssync_receiver.py +19 -9
- swift/obj/ssync_sender.py +10 -10
- swift/proxy/controllers/account.py +7 -7
- swift/proxy/controllers/base.py +374 -366
- swift/proxy/controllers/container.py +112 -53
- swift/proxy/controllers/obj.py +254 -390
- swift/proxy/server.py +3 -8
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-server +1 -1
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-server +1 -1
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-drive-audit +45 -14
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-server +1 -1
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-proxy-server +1 -1
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/AUTHORS +4 -0
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/METADATA +32 -35
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/RECORD +103 -100
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/WHEEL +1 -1
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/entry_points.txt +0 -1
- swift-2.32.1.dist-info/pbr.json +1 -0
- swift-2.31.1.dist-info/pbr.json +0 -1
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-audit +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-auditor +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-info +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-reaper +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-account-replicator +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-config +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-auditor +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-info +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-reconciler +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-replicator +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-sharder +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-sync +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-container-updater +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-dispersion-populate +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-dispersion-report +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-form-signature +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-get-nodes +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-init +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-auditor +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-expirer +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-info +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-reconstructor +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-relinker +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-replicator +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-object-updater +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-oldies +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-orphans +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-recon +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-recon-cron +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-reconciler-enqueue +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-builder +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-builder-analyzer +0 -0
- {swift-2.31.1.data → swift-2.32.1.data}/scripts/swift-ring-composer +0 -0
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/LICENSE +0 -0
- {swift-2.31.1.dist-info → swift-2.32.1.dist-info}/top_level.txt +0 -0
@@ -23,20 +23,24 @@ class CrossDomainMiddleware(object):
|
|
23
23
|
Cross domain middleware used to respond to requests for cross domain
|
24
24
|
policy information.
|
25
25
|
|
26
|
-
If the path is
|
27
|
-
policy document. This allows web pages hosted elsewhere to use
|
28
|
-
side technologies such as Flash, Java and Silverlight to interact
|
26
|
+
If the path is ``/crossdomain.xml`` it will respond with an xml cross
|
27
|
+
domain policy document. This allows web pages hosted elsewhere to use
|
28
|
+
client side technologies such as Flash, Java and Silverlight to interact
|
29
29
|
with the Swift API.
|
30
30
|
|
31
31
|
To enable this middleware, add it to the pipeline in your proxy-server.conf
|
32
32
|
file. It should be added before any authentication (e.g., tempauth or
|
33
33
|
keystone) middleware. In this example ellipsis (...) indicate other
|
34
|
-
middleware you may have chosen to use
|
34
|
+
middleware you may have chosen to use:
|
35
|
+
|
36
|
+
.. code:: cfg
|
35
37
|
|
36
38
|
[pipeline:main]
|
37
39
|
pipeline = ... crossdomain ... authtoken ... proxy-server
|
38
40
|
|
39
|
-
And add a filter section, such as
|
41
|
+
And add a filter section, such as:
|
42
|
+
|
43
|
+
.. code:: cfg
|
40
44
|
|
41
45
|
[filter:crossdomain]
|
42
46
|
use = egg:swift#crossdomain
|
@@ -45,13 +49,22 @@ class CrossDomainMiddleware(object):
|
|
45
49
|
|
46
50
|
For continuation lines, put some whitespace before the continuation
|
47
51
|
text. Ensure you put a completely blank line to terminate the
|
48
|
-
cross_domain_policy value.
|
52
|
+
``cross_domain_policy`` value.
|
53
|
+
|
54
|
+
The ``cross_domain_policy`` name/value is optional. If omitted, the policy
|
55
|
+
defaults as if you had specified:
|
49
56
|
|
50
|
-
|
51
|
-
defaults as if you had specified::
|
57
|
+
.. code:: cfg
|
52
58
|
|
53
59
|
cross_domain_policy = <allow-access-from domain="*" secure="false" />
|
54
60
|
|
61
|
+
.. note::
|
62
|
+
|
63
|
+
The default policy is very permissive; this is appropriate
|
64
|
+
for most public cloud deployments, but may not be appropriate
|
65
|
+
for all deployments. See also:
|
66
|
+
`CWE-942 <https://cwe.mitre.org/data/definitions/942.html>`__
|
67
|
+
|
55
68
|
|
56
69
|
"""
|
57
70
|
|
@@ -18,6 +18,7 @@ import six
|
|
18
18
|
from xml.etree.cElementTree import Element, SubElement, tostring
|
19
19
|
|
20
20
|
from swift.common.constraints import valid_api_version
|
21
|
+
from swift.common.header_key_dict import HeaderKeyDict
|
21
22
|
from swift.common.http import HTTP_NO_CONTENT
|
22
23
|
from swift.common.request_helpers import get_param
|
23
24
|
from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
|
@@ -178,52 +179,39 @@ class ListingFilter(object):
|
|
178
179
|
start_response(status, headers)
|
179
180
|
return resp_iter
|
180
181
|
|
181
|
-
header_to_index = {}
|
182
|
-
resp_content_type = resp_length = None
|
183
|
-
for i, (header, value) in enumerate(headers):
|
184
|
-
header = header.lower()
|
185
|
-
if header == 'content-type':
|
186
|
-
header_to_index[header] = i
|
187
|
-
resp_content_type = value.partition(';')[0]
|
188
|
-
elif header == 'content-length':
|
189
|
-
header_to_index[header] = i
|
190
|
-
resp_length = int(value)
|
191
|
-
elif header == 'vary':
|
192
|
-
header_to_index[header] = i
|
193
|
-
|
194
182
|
if not status.startswith(('200 ', '204 ')):
|
195
183
|
start_response(status, headers)
|
196
184
|
return resp_iter
|
197
185
|
|
186
|
+
headers_dict = HeaderKeyDict(headers)
|
187
|
+
resp_content_type = headers_dict.get(
|
188
|
+
'content-type', '').partition(';')[0]
|
189
|
+
resp_length = headers_dict.get('content-length')
|
190
|
+
|
198
191
|
if can_vary:
|
199
|
-
if 'vary' in
|
200
|
-
value =
|
192
|
+
if 'vary' in headers_dict:
|
193
|
+
value = headers_dict['vary']
|
201
194
|
if 'accept' not in list_from_csv(value.lower()):
|
202
|
-
|
203
|
-
'Vary', value + ', Accept')
|
195
|
+
headers_dict['vary'] = value + ', Accept'
|
204
196
|
else:
|
205
|
-
|
197
|
+
headers_dict['vary'] = 'Accept'
|
206
198
|
|
207
199
|
if resp_content_type != 'application/json':
|
208
|
-
start_response(status,
|
200
|
+
start_response(status, list(headers_dict.items()))
|
209
201
|
return resp_iter
|
210
202
|
|
211
|
-
if
|
212
|
-
|
213
|
-
|
203
|
+
if req.method == 'HEAD':
|
204
|
+
headers_dict['content-type'] = out_content_type + '; charset=utf-8'
|
205
|
+
# proxy logging (and maybe other mw?) seem to be good about
|
206
|
+
# sticking this on HEAD/204 but we do it here to be responsible
|
207
|
+
# and explicit
|
208
|
+
headers_dict['content-length'] = 0
|
209
|
+
start_response(status, list(headers_dict.items()))
|
214
210
|
return resp_iter
|
215
211
|
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
else:
|
220
|
-
headers[header_to_index[header]] = (
|
221
|
-
headers[header_to_index[header]][0], str(value))
|
222
|
-
|
223
|
-
if req.method == 'HEAD':
|
224
|
-
set_header('content-type', out_content_type + '; charset=utf-8')
|
225
|
-
set_header('content-length', None) # don't know, can't determine
|
226
|
-
start_response(status, headers)
|
212
|
+
if resp_length is None or \
|
213
|
+
int(resp_length) > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
|
214
|
+
start_response(status, list(headers_dict.items()))
|
227
215
|
return resp_iter
|
228
216
|
|
229
217
|
body = b''.join(resp_iter)
|
@@ -237,7 +225,7 @@ class ListingFilter(object):
|
|
237
225
|
except ValueError:
|
238
226
|
# Static web listing that's returning invalid JSON?
|
239
227
|
# Just pass it straight through; that's about all we *can* do.
|
240
|
-
start_response(status,
|
228
|
+
start_response(status, list(headers_dict.items()))
|
241
229
|
return [body]
|
242
230
|
|
243
231
|
if not req.allow_reserved_names:
|
@@ -257,16 +245,16 @@ class ListingFilter(object):
|
|
257
245
|
body = json.dumps(listing).encode('ascii')
|
258
246
|
except KeyError:
|
259
247
|
# listing was in a bad format -- funky static web listing??
|
260
|
-
start_response(status,
|
248
|
+
start_response(status, list(headers_dict.items()))
|
261
249
|
return [body]
|
262
250
|
|
263
251
|
if not body:
|
264
252
|
status = '%s %s' % (HTTP_NO_CONTENT,
|
265
253
|
RESPONSE_REASONS[HTTP_NO_CONTENT][0])
|
266
254
|
|
267
|
-
|
268
|
-
|
269
|
-
start_response(status,
|
255
|
+
headers_dict['content-type'] = out_content_type + '; charset=utf-8'
|
256
|
+
headers_dict['content-length'] = len(body)
|
257
|
+
start_response(status, list(headers_dict.items()))
|
270
258
|
return [body]
|
271
259
|
|
272
260
|
|
@@ -19,12 +19,12 @@ Logging middleware for the Swift proxy.
|
|
19
19
|
This serves as both the default logging implementation and an example of how
|
20
20
|
to plug in your own logging format/method.
|
21
21
|
|
22
|
-
The logging format implemented below is as follows
|
22
|
+
The logging format implemented below is as follows::
|
23
23
|
|
24
|
-
client_ip remote_addr end_time.datetime method path protocol
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
client_ip remote_addr end_time.datetime method path protocol
|
25
|
+
status_int referer user_agent auth_token bytes_recvd bytes_sent
|
26
|
+
client_etag transaction_id headers request_time source log_info
|
27
|
+
start_time end_time policy_index
|
28
28
|
|
29
29
|
These values are space-separated, and each is url-encoded, so that they can
|
30
30
|
be separated with a simple .split()
|
@@ -49,6 +49,11 @@ be separated with a simple .split()
|
|
49
49
|
* Values that are missing (e.g. due to a header not being present) or zero
|
50
50
|
are generally represented by a single hyphen ('-').
|
51
51
|
|
52
|
+
.. note::
|
53
|
+
The message format may be configured using the ``log_msg_template`` option,
|
54
|
+
allowing fields to be added, removed, re-ordered, and even anonymized. For
|
55
|
+
more information, see https://docs.openstack.org/swift/latest/logs.html
|
56
|
+
|
52
57
|
The proxy-logging can be used twice in the proxy server's pipeline when there
|
53
58
|
is middleware installed that can return custom responses that don't follow the
|
54
59
|
standard pipeline to the proxy server.
|
@@ -78,7 +83,7 @@ from swift.common.middleware.catch_errors import enforce_byte_count
|
|
78
83
|
from swift.common.swob import Request
|
79
84
|
from swift.common.utils import (get_logger, get_remote_client,
|
80
85
|
config_true_value, reiterate,
|
81
|
-
close_if_possible,
|
86
|
+
close_if_possible, cap_length,
|
82
87
|
InputProxy, list_from_csv, get_policy_index,
|
83
88
|
split_path, StrAnonymizer, StrFormatTime,
|
84
89
|
LogStringFormatter)
|
@@ -200,9 +205,7 @@ class ProxyLoggingMiddleware(object):
|
|
200
205
|
env['swift.proxy_access_log_made'] = True
|
201
206
|
|
202
207
|
def obscure_sensitive(self, value):
|
203
|
-
|
204
|
-
return value[:self.reveal_sensitive_prefix] + '...'
|
205
|
-
return value
|
208
|
+
return cap_length(value, self.reveal_sensitive_prefix)
|
206
209
|
|
207
210
|
def obscure_req(self, req):
|
208
211
|
for header in get_sensitive_headers():
|
@@ -23,7 +23,8 @@ from swift.common import swob
|
|
23
23
|
from swift.common.http import HTTP_OK
|
24
24
|
from swift.common.middleware.versioned_writes.object_versioning import \
|
25
25
|
DELETE_MARKER_CONTENT_TYPE
|
26
|
-
from swift.common.utils import json, public, config_true_value, Timestamp
|
26
|
+
from swift.common.utils import json, public, config_true_value, Timestamp, \
|
27
|
+
cap_length
|
27
28
|
from swift.common.registry import get_swift_info
|
28
29
|
|
29
30
|
from swift.common.middleware.s3api.controllers.base import Controller
|
@@ -343,7 +344,12 @@ class BucketController(Controller):
|
|
343
344
|
|
344
345
|
resp = req.get_response(self.app, query=query)
|
345
346
|
|
346
|
-
|
347
|
+
try:
|
348
|
+
objects = json.loads(resp.body)
|
349
|
+
except (TypeError, ValueError):
|
350
|
+
self.logger.error('Got non-JSON response trying to list %s: %r',
|
351
|
+
req.path, cap_length(resp.body, 60))
|
352
|
+
raise
|
347
353
|
|
348
354
|
is_truncated = max_keys > 0 and len(objects) > max_keys
|
349
355
|
objects = objects[:max_keys]
|
@@ -151,6 +151,7 @@ from swift.common.middleware.listing_formats import \
|
|
151
151
|
MAX_CONTAINER_LISTING_CONTENT_LENGTH
|
152
152
|
from swift.common.wsgi import PipelineWrapper, loadcontext, WSGIContext
|
153
153
|
|
154
|
+
from swift.common.middleware import app_property
|
154
155
|
from swift.common.middleware.s3api.exception import NotS3Request, \
|
155
156
|
InvalidSubresource
|
156
157
|
from swift.common.middleware.s3api.s3request import get_request_class
|
@@ -167,9 +168,12 @@ from swift.common.registry import register_swift_info, \
|
|
167
168
|
class ListingEtagMiddleware(object):
|
168
169
|
def __init__(self, app):
|
169
170
|
self.app = app
|
170
|
-
|
171
|
-
|
172
|
-
|
171
|
+
|
172
|
+
# Pass these along so get_container_info will have the configured
|
173
|
+
# odds to skip cache
|
174
|
+
_pipeline_final_app = app_property('_pipeline_final_app')
|
175
|
+
_pipeline_request_logging_app = app_property(
|
176
|
+
'_pipeline_request_logging_app')
|
173
177
|
|
174
178
|
def __call__(self, env, start_response):
|
175
179
|
# a lot of this is cribbed from listing_formats / swob.Request
|
@@ -293,7 +297,7 @@ class S3ApiMiddleware(object):
|
|
293
297
|
wsgi_conf.get('ratelimit_as_client_error', False))
|
294
298
|
|
295
299
|
self.logger = get_logger(
|
296
|
-
wsgi_conf, log_route=
|
300
|
+
wsgi_conf, log_route='s3api', statsd_tail_prefix='s3api')
|
297
301
|
self.check_pipeline(wsgi_conf)
|
298
302
|
|
299
303
|
def is_s3_cors_preflight(self, env):
|
@@ -348,6 +352,7 @@ class S3ApiMiddleware(object):
|
|
348
352
|
except InvalidSubresource as e:
|
349
353
|
self.logger.debug(e.cause)
|
350
354
|
except ErrorResponse as err_resp:
|
355
|
+
self.logger.increment(err_resp.metric_name)
|
351
356
|
if isinstance(err_resp, InternalError):
|
352
357
|
self.logger.exception(err_resp)
|
353
358
|
resp = err_resp
|
@@ -189,11 +189,13 @@ class SigV4Mixin(object):
|
|
189
189
|
timestamp = mktime(self.headers.get('Date'))
|
190
190
|
except (ValueError, TypeError):
|
191
191
|
raise AccessDenied('AWS authentication requires a valid Date '
|
192
|
-
'or x-amz-date header'
|
192
|
+
'or x-amz-date header',
|
193
|
+
reason='invalid_date')
|
193
194
|
|
194
195
|
if timestamp < 0:
|
195
196
|
raise AccessDenied('AWS authentication requires a valid Date '
|
196
|
-
'or x-amz-date header'
|
197
|
+
'or x-amz-date header',
|
198
|
+
reason='invalid_date')
|
197
199
|
|
198
200
|
try:
|
199
201
|
self._timestamp = S3Timestamp(timestamp)
|
@@ -214,7 +216,7 @@ class SigV4Mixin(object):
|
|
214
216
|
try:
|
215
217
|
expires = int(self.params['X-Amz-Expires'])
|
216
218
|
except KeyError:
|
217
|
-
raise AccessDenied()
|
219
|
+
raise AccessDenied(reason='invalid_expires')
|
218
220
|
except ValueError:
|
219
221
|
err = 'X-Amz-Expires should be a number'
|
220
222
|
else:
|
@@ -230,14 +232,14 @@ class SigV4Mixin(object):
|
|
230
232
|
raise AuthorizationQueryParametersError(err)
|
231
233
|
|
232
234
|
if int(self.timestamp) + expires < S3Timestamp.now():
|
233
|
-
raise AccessDenied('Request has expired')
|
235
|
+
raise AccessDenied('Request has expired', reason='expired')
|
234
236
|
|
235
237
|
def _parse_credential(self, credential_string):
|
236
238
|
parts = credential_string.split("/")
|
237
239
|
# credential must be in following format:
|
238
240
|
# <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request
|
239
241
|
if not parts[0] or len(parts) != 5:
|
240
|
-
raise AccessDenied()
|
242
|
+
raise AccessDenied(reason='invalid_credential')
|
241
243
|
return dict(zip(['access', 'date', 'region', 'service', 'terminal'],
|
242
244
|
parts))
|
243
245
|
|
@@ -257,9 +259,9 @@ class SigV4Mixin(object):
|
|
257
259
|
swob.wsgi_to_str(self.params['X-Amz-Credential']))
|
258
260
|
sig = swob.wsgi_to_str(self.params['X-Amz-Signature'])
|
259
261
|
if not sig:
|
260
|
-
raise AccessDenied()
|
262
|
+
raise AccessDenied(reason='invalid_query_auth')
|
261
263
|
except KeyError:
|
262
|
-
raise AccessDenied()
|
264
|
+
raise AccessDenied(reason='invalid_query_auth')
|
263
265
|
|
264
266
|
try:
|
265
267
|
signed_headers = swob.wsgi_to_str(
|
@@ -311,7 +313,7 @@ class SigV4Mixin(object):
|
|
311
313
|
"Credential=")[2].split(',')[0])
|
312
314
|
sig = auth_str.partition("Signature=")[2].split(',')[0]
|
313
315
|
if not sig:
|
314
|
-
raise AccessDenied()
|
316
|
+
raise AccessDenied(reason='invalid_header_auth')
|
315
317
|
signed_headers = auth_str.partition(
|
316
318
|
"SignedHeaders=")[2].split(',', 1)[0]
|
317
319
|
if not signed_headers:
|
@@ -582,11 +584,13 @@ class S3Request(swob.Request):
|
|
582
584
|
self.headers.get('Date')))
|
583
585
|
except ValueError:
|
584
586
|
raise AccessDenied('AWS authentication requires a valid Date '
|
585
|
-
'or x-amz-date header'
|
587
|
+
'or x-amz-date header',
|
588
|
+
reason='invalid_date')
|
586
589
|
|
587
590
|
if timestamp < 0:
|
588
591
|
raise AccessDenied('AWS authentication requires a valid Date '
|
589
|
-
'or x-amz-date header'
|
592
|
+
'or x-amz-date header',
|
593
|
+
reason='invalid_date')
|
590
594
|
try:
|
591
595
|
self._timestamp = S3Timestamp(timestamp)
|
592
596
|
except ValueError:
|
@@ -658,10 +662,10 @@ class S3Request(swob.Request):
|
|
658
662
|
expires = swob.wsgi_to_str(self.params['Expires'])
|
659
663
|
sig = swob.wsgi_to_str(self.params['Signature'])
|
660
664
|
except KeyError:
|
661
|
-
raise AccessDenied()
|
665
|
+
raise AccessDenied(reason='invalid_query_auth')
|
662
666
|
|
663
667
|
if not all([access, sig, expires]):
|
664
|
-
raise AccessDenied()
|
668
|
+
raise AccessDenied(reason='invalid_query_auth')
|
665
669
|
|
666
670
|
return access, sig
|
667
671
|
|
@@ -674,7 +678,7 @@ class S3Request(swob.Request):
|
|
674
678
|
"""
|
675
679
|
auth_str = swob.wsgi_to_str(self.headers['Authorization'])
|
676
680
|
if not auth_str.startswith('AWS ') or ':' not in auth_str:
|
677
|
-
raise AccessDenied()
|
681
|
+
raise AccessDenied(reason='invalid_header_auth')
|
678
682
|
# This means signature format V2
|
679
683
|
access, sig = auth_str.split(' ', 1)[1].rsplit(':', 1)
|
680
684
|
return access, sig
|
@@ -705,15 +709,15 @@ class S3Request(swob.Request):
|
|
705
709
|
try:
|
706
710
|
ex = S3Timestamp(float(self.params['Expires']))
|
707
711
|
except (KeyError, ValueError):
|
708
|
-
raise AccessDenied()
|
712
|
+
raise AccessDenied(reason='invalid_expires')
|
709
713
|
|
710
714
|
if S3Timestamp.now() > ex:
|
711
|
-
raise AccessDenied('Request has expired')
|
715
|
+
raise AccessDenied('Request has expired', reason='expired')
|
712
716
|
|
713
717
|
if ex >= 2 ** 31:
|
714
718
|
raise AccessDenied(
|
715
719
|
'Invalid date (should be seconds since epoch): %s' %
|
716
|
-
self.params['Expires'])
|
720
|
+
self.params['Expires'], reason='invalid_expires')
|
717
721
|
|
718
722
|
def _validate_dates(self):
|
719
723
|
"""
|
@@ -725,12 +729,13 @@ class S3Request(swob.Request):
|
|
725
729
|
amz_date_header = self.headers.get('X-Amz-Date')
|
726
730
|
if not date_header and not amz_date_header:
|
727
731
|
raise AccessDenied('AWS authentication requires a valid Date '
|
728
|
-
'or x-amz-date header'
|
732
|
+
'or x-amz-date header',
|
733
|
+
reason='invalid_date')
|
729
734
|
|
730
735
|
# Anyways, request timestamp should be validated
|
731
736
|
epoch = S3Timestamp(0)
|
732
737
|
if self.timestamp < epoch:
|
733
|
-
raise AccessDenied()
|
738
|
+
raise AccessDenied(reason='invalid_date')
|
734
739
|
|
735
740
|
# If the standard date is too far ahead or behind, it is an
|
736
741
|
# error
|
@@ -984,7 +989,7 @@ class S3Request(swob.Request):
|
|
984
989
|
else:
|
985
990
|
# Should have already raised NotS3Request in _parse_auth_info,
|
986
991
|
# but as a sanity check...
|
987
|
-
raise AccessDenied()
|
992
|
+
raise AccessDenied(reason='not_s3')
|
988
993
|
|
989
994
|
for key, value in sorted(amz_headers.items()):
|
990
995
|
buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
|
@@ -1348,15 +1353,18 @@ class S3Request(swob.Request):
|
|
1348
1353
|
try:
|
1349
1354
|
sw_resp = sw_req.get_response(app)
|
1350
1355
|
except swob.HTTPException as err:
|
1356
|
+
# Maybe a 422 from HashingInput? Put something in
|
1357
|
+
# s3api.backend_path - hopefully by now any modifications to the
|
1358
|
+
# path (e.g. tenant to account translation) will have been made by
|
1359
|
+
# auth middleware
|
1360
|
+
self.environ['s3api.backend_path'] = sw_req.environ['PATH_INFO']
|
1351
1361
|
sw_resp = err
|
1352
1362
|
else:
|
1353
1363
|
# reuse account
|
1354
1364
|
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
|
1355
1365
|
2, 3, True)
|
1356
|
-
#
|
1357
|
-
|
1358
|
-
self.environ['s3api.backend_path'] = \
|
1359
|
-
sw_resp.environ['PATH_INFO']
|
1366
|
+
# Update s3.backend_path from the response environ
|
1367
|
+
self.environ['s3api.backend_path'] = sw_resp.environ['PATH_INFO']
|
1360
1368
|
# Propogate backend headers back into our req headers for logging
|
1361
1369
|
for k, v in sw_req.headers.items():
|
1362
1370
|
if k.lower().startswith('x-backend-'):
|
@@ -1412,7 +1420,7 @@ class S3Request(swob.Request):
|
|
1412
1420
|
raise SignatureDoesNotMatch(
|
1413
1421
|
**self.signature_does_not_match_kwargs())
|
1414
1422
|
if status == HTTP_FORBIDDEN:
|
1415
|
-
raise AccessDenied()
|
1423
|
+
raise AccessDenied(reason='forbidden')
|
1416
1424
|
if status == HTTP_SERVICE_UNAVAILABLE:
|
1417
1425
|
raise ServiceUnavailable()
|
1418
1426
|
if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):
|
@@ -229,11 +229,12 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
|
|
229
229
|
_code = ''
|
230
230
|
xml_declaration = True
|
231
231
|
|
232
|
-
def __init__(self, msg=None, *args, **kwargs):
|
232
|
+
def __init__(self, msg=None, reason=None, *args, **kwargs):
|
233
233
|
if msg:
|
234
234
|
self._msg = msg
|
235
235
|
if not self._code:
|
236
236
|
self._code = self.__class__.__name__
|
237
|
+
self.reason = reason
|
237
238
|
|
238
239
|
self.info = kwargs.copy()
|
239
240
|
for reserved_key in ('headers', 'body'):
|
@@ -247,6 +248,14 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
|
|
247
248
|
**kwargs)
|
248
249
|
self.headers = HeaderKeyDict(self.headers)
|
249
250
|
|
251
|
+
@property
|
252
|
+
def metric_name(self):
|
253
|
+
parts = [str(self.status_int), self._code]
|
254
|
+
if self.reason:
|
255
|
+
parts.append(self.reason)
|
256
|
+
metric = '.'.join(parts)
|
257
|
+
return metric.replace(' ', '_')
|
258
|
+
|
250
259
|
def _body_iter(self):
|
251
260
|
error_elem = Element('Error')
|
252
261
|
SubElement(error_elem, 'Code').text = self._code
|
@@ -804,24 +804,23 @@ class TempAuth(object):
|
|
804
804
|
key = req.headers.get('x-storage-pass')
|
805
805
|
else:
|
806
806
|
return HTTPBadRequest(request=req)
|
807
|
+
unauthed_headers = {
|
808
|
+
'Www-Authenticate': 'Swift realm="%s"' % (account or 'unknown'),
|
809
|
+
}
|
807
810
|
if not all((account, user, key)):
|
808
811
|
self.logger.increment('token_denied')
|
809
|
-
|
810
|
-
return HTTPUnauthorized(request=req, headers={'Www-Authenticate':
|
811
|
-
'Swift realm="%s"' %
|
812
|
-
realm})
|
812
|
+
return HTTPUnauthorized(request=req, headers=unauthed_headers)
|
813
813
|
# Authenticate user
|
814
|
+
account = wsgi_to_str(account)
|
815
|
+
user = wsgi_to_str(user)
|
816
|
+
key = wsgi_to_str(key)
|
814
817
|
account_user = account + ':' + user
|
815
818
|
if account_user not in self.users:
|
816
819
|
self.logger.increment('token_denied')
|
817
|
-
|
818
|
-
return HTTPUnauthorized(request=req,
|
819
|
-
headers={'Www-Authenticate': auth})
|
820
|
+
return HTTPUnauthorized(request=req, headers=unauthed_headers)
|
820
821
|
if self.users[account_user]['key'] != key:
|
821
822
|
self.logger.increment('token_denied')
|
822
|
-
|
823
|
-
return HTTPUnauthorized(request=req,
|
824
|
-
headers={'Www-Authenticate': auth})
|
823
|
+
return HTTPUnauthorized(request=req, headers=unauthed_headers)
|
825
824
|
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
|
826
825
|
# Get memcache client
|
827
826
|
memcache_client = cache_from_env(req.environ)
|
@@ -47,8 +47,5 @@ def filter_factory(global_conf, **local_conf):
|
|
47
47
|
if 'symlink' not in get_swift_info():
|
48
48
|
raise ValueError('object versioning requires symlinks')
|
49
49
|
app = ObjectVersioningMiddleware(app, conf)
|
50
|
-
# Pass this along so get_container_info will have the configured
|
51
|
-
# odds to skip cache
|
52
|
-
app._pipeline_final_app = app.app._pipeline_final_app
|
53
50
|
return VersionedWritesMiddleware(app, conf)
|
54
51
|
return versioning_filter
|
@@ -158,6 +158,7 @@ from swift.common.http import is_success, is_client_error, HTTP_NOT_FOUND, \
|
|
158
158
|
from swift.common.request_helpers import get_sys_meta_prefix, \
|
159
159
|
copy_header_subset, get_reserved_name, split_reserved_name, \
|
160
160
|
constrain_req_limit
|
161
|
+
from swift.common.middleware import app_property
|
161
162
|
from swift.common.middleware.symlink import TGT_OBJ_SYMLINK_HDR, \
|
162
163
|
TGT_ETAG_SYSMETA_SYMLINK_HDR, SYMLOOP_EXTEND, ALLOW_RESERVED_NAMES, \
|
163
164
|
TGT_BYTES_SYSMETA_SYMLINK_HDR, TGT_ACCT_SYMLINK_HDR
|
@@ -1165,12 +1166,22 @@ class ContainerContext(ObjectVersioningContext):
|
|
1165
1166
|
params['prefix'] = get_reserved_name(params['prefix'])
|
1166
1167
|
|
1167
1168
|
# NB: no end_marker support (yet)
|
1168
|
-
versions_req.
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1169
|
+
if get_container_info(versions_req.environ, self.app,
|
1170
|
+
swift_source='OV')['status'] == 404:
|
1171
|
+
# we don't usually like to LBYL like this, but 404s tend to be
|
1172
|
+
# expensive (since we check all primaries and a bunch of handoffs)
|
1173
|
+
# and we expect this to be a reasonably common way to listing
|
1174
|
+
# objects since it's more complete from the user's perspective
|
1175
|
+
# (see also: s3api and that client ecosystem)
|
1176
|
+
versions_resp = None
|
1177
|
+
else:
|
1178
|
+
versions_req.params = {
|
1179
|
+
k: params.get(k, '') for k in (
|
1180
|
+
'prefix', 'marker', 'limit', 'delimiter', 'reverse')}
|
1181
|
+
versions_resp = versions_req.get_response(self.app)
|
1172
1182
|
|
1173
|
-
if versions_resp
|
1183
|
+
if versions_resp is None \
|
1184
|
+
or versions_resp.status_int == HTTP_NOT_FOUND:
|
1174
1185
|
subdir_listing = [{'subdir': s} for s in subdir_set]
|
1175
1186
|
broken_listing = []
|
1176
1187
|
for item in current_versions.values():
|
@@ -1379,6 +1390,12 @@ class ObjectVersioningMiddleware(object):
|
|
1379
1390
|
self.conf = conf
|
1380
1391
|
self.logger = get_logger(conf, log_route='object_versioning')
|
1381
1392
|
|
1393
|
+
# Pass these along so get_container_info will have the configured
|
1394
|
+
# odds to skip cache
|
1395
|
+
_pipeline_final_app = app_property('_pipeline_final_app')
|
1396
|
+
_pipeline_request_logging_app = app_property(
|
1397
|
+
'_pipeline_request_logging_app')
|
1398
|
+
|
1382
1399
|
def account_request(self, req, api_version, account, start_response):
|
1383
1400
|
account_ctx = AccountContext(self.app, self.logger)
|
1384
1401
|
if req.method == 'GET':
|
@@ -246,7 +246,7 @@ class HTMLViewer(object):
|
|
246
246
|
if multiple:
|
247
247
|
return value
|
248
248
|
if isinstance(value, list):
|
249
|
-
return
|
249
|
+
return int(value[0]) if isinstance(default, int) else value[0]
|
250
250
|
else:
|
251
251
|
return value
|
252
252
|
|
@@ -16,6 +16,11 @@
|
|
16
16
|
"""
|
17
17
|
Profiling middleware for Swift Servers.
|
18
18
|
|
19
|
+
.. note::
|
20
|
+
This middleware is intended for development and testing environments only,
|
21
|
+
not production. No authentication is expected or required for the web UI,
|
22
|
+
and profiling may incur noticeable performance penalties.
|
23
|
+
|
19
24
|
The current implementation is based on eventlet aware profiler.(For the
|
20
25
|
future, more profilers could be added in to collect more data for analysis.)
|
21
26
|
Profiling all incoming requests and accumulating cpu timing statistics
|
swift/common/request_helpers.py
CHANGED
@@ -27,7 +27,6 @@ import time
|
|
27
27
|
import six
|
28
28
|
from swift.common.header_key_dict import HeaderKeyDict
|
29
29
|
|
30
|
-
from swift import gettext_ as _
|
31
30
|
from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX, \
|
32
31
|
CONTAINER_LISTING_LIMIT
|
33
32
|
from swift.common.storage_policy import POLICIES
|
@@ -202,7 +201,7 @@ def get_name_and_placement(request, minsegs=1, maxsegs=None,
|
|
202
201
|
policy = POLICIES.get_by_index(policy_index)
|
203
202
|
if not policy:
|
204
203
|
raise HTTPServiceUnavailable(
|
205
|
-
body=
|
204
|
+
body="No policy with index %s" % policy_index,
|
206
205
|
request=request, content_type='text/plain')
|
207
206
|
results = split_and_validate_path(request, minsegs=minsegs,
|
208
207
|
maxsegs=maxsegs,
|