swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/__init__.py +29 -50
- swift/account/auditor.py +21 -118
- swift/account/backend.py +33 -28
- swift/account/reaper.py +37 -28
- swift/account/replicator.py +22 -0
- swift/account/server.py +60 -26
- swift/account/utils.py +28 -11
- swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
- swift-2.23.3.data/scripts/swift-config → swift/cli/config.py +2 -2
- swift/cli/container_deleter.py +5 -11
- swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
- swift/cli/dispersion_report.py +10 -9
- swift-2.23.3.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
- swift/cli/form_signature.py +3 -7
- swift-2.23.3.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
- swift/cli/info.py +154 -14
- swift/cli/manage_shard_ranges.py +705 -37
- swift-2.23.3.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
- swift-2.23.3.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
- swift/cli/recon.py +196 -67
- swift-2.23.3.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
- swift-2.23.3.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +807 -126
- swift/cli/reload.py +135 -0
- swift/cli/ringbuilder.py +217 -20
- swift/cli/ringcomposer.py +0 -1
- swift/cli/shard-info.py +4 -3
- swift/common/base_storage_server.py +9 -20
- swift/common/bufferedhttp.py +48 -74
- swift/common/constraints.py +20 -15
- swift/common/container_sync_realms.py +9 -11
- swift/common/daemon.py +25 -8
- swift/common/db.py +195 -128
- swift/common/db_auditor.py +168 -0
- swift/common/db_replicator.py +95 -55
- swift/common/digest.py +141 -0
- swift/common/direct_client.py +144 -33
- swift/common/error_limiter.py +93 -0
- swift/common/exceptions.py +25 -1
- swift/common/header_key_dict.py +2 -9
- swift/common/http_protocol.py +373 -0
- swift/common/internal_client.py +129 -59
- swift/common/linkat.py +3 -4
- swift/common/manager.py +284 -67
- swift/common/memcached.py +390 -145
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +211 -46
- swift/common/middleware/acl.py +3 -8
- swift/common/middleware/backend_ratelimit.py +230 -0
- swift/common/middleware/bulk.py +22 -34
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +6 -11
- swift/common/middleware/container_quotas.py +1 -1
- swift/common/middleware/container_sync.py +39 -17
- swift/common/middleware/copy.py +12 -0
- swift/common/middleware/crossdomain.py +22 -9
- swift/common/middleware/crypto/__init__.py +2 -1
- swift/common/middleware/crypto/crypto_utils.py +11 -15
- swift/common/middleware/crypto/decrypter.py +28 -11
- swift/common/middleware/crypto/encrypter.py +12 -17
- swift/common/middleware/crypto/keymaster.py +8 -15
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/dlo.py +15 -11
- swift/common/middleware/domain_remap.py +5 -4
- swift/common/middleware/etag_quoter.py +128 -0
- swift/common/middleware/formpost.py +73 -70
- swift/common/middleware/gatekeeper.py +8 -1
- swift/common/middleware/keystoneauth.py +33 -3
- swift/common/middleware/list_endpoints.py +4 -4
- swift/common/middleware/listing_formats.py +85 -49
- swift/common/middleware/memcache.py +4 -95
- swift/common/middleware/name_check.py +3 -2
- swift/common/middleware/proxy_logging.py +160 -92
- swift/common/middleware/ratelimit.py +17 -10
- swift/common/middleware/read_only.py +6 -4
- swift/common/middleware/recon.py +59 -22
- swift/common/middleware/s3api/acl_handlers.py +25 -3
- swift/common/middleware/s3api/acl_utils.py +6 -1
- swift/common/middleware/s3api/controllers/__init__.py +6 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/bucket.py +242 -137
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
- swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
- swift/common/middleware/s3api/controllers/obj.py +112 -8
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
- swift/common/middleware/s3api/controllers/tagging.py +57 -0
- swift/common/middleware/s3api/controllers/versioning.py +36 -7
- swift/common/middleware/s3api/etree.py +22 -9
- swift/common/middleware/s3api/exception.py +0 -4
- swift/common/middleware/s3api/s3api.py +113 -41
- swift/common/middleware/s3api/s3request.py +384 -218
- swift/common/middleware/s3api/s3response.py +126 -23
- swift/common/middleware/s3api/s3token.py +16 -17
- swift/common/middleware/s3api/schema/delete.rng +1 -1
- swift/common/middleware/s3api/subresource.py +7 -10
- swift/common/middleware/s3api/utils.py +27 -10
- swift/common/middleware/slo.py +665 -358
- swift/common/middleware/staticweb.py +64 -37
- swift/common/middleware/symlink.py +51 -18
- swift/common/middleware/tempauth.py +76 -58
- swift/common/middleware/tempurl.py +191 -173
- swift/common/middleware/versioned_writes/__init__.py +51 -0
- swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
- swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +18 -19
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +10 -10
- swift-2.23.3.data/scripts/swift-container-server → swift/common/recon.py +13 -8
- swift/common/registry.py +147 -0
- swift/common/request_helpers.py +324 -57
- swift/common/ring/builder.py +67 -25
- swift/common/ring/composite_builder.py +1 -1
- swift/common/ring/ring.py +177 -51
- swift/common/ring/utils.py +1 -1
- swift/common/splice.py +10 -6
- swift/common/statsd_client.py +205 -0
- swift/common/storage_policy.py +49 -44
- swift/common/swob.py +86 -102
- swift/common/{utils.py → utils/__init__.py} +2163 -2772
- swift/common/utils/base.py +131 -0
- swift/common/utils/config.py +433 -0
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/logs.py +859 -0
- swift/common/utils/timestamp.py +412 -0
- swift/common/wsgi.py +553 -535
- swift/container/auditor.py +14 -100
- swift/container/backend.py +490 -231
- swift/container/reconciler.py +126 -37
- swift/container/replicator.py +96 -22
- swift/container/server.py +358 -165
- swift/container/sharder.py +1540 -684
- swift/container/sync.py +94 -88
- swift/container/updater.py +53 -32
- swift/obj/auditor.py +153 -35
- swift/obj/diskfile.py +466 -217
- swift/obj/expirer.py +406 -124
- swift/obj/mem_diskfile.py +7 -4
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +523 -262
- swift/obj/replicator.py +249 -188
- swift/obj/server.py +207 -122
- swift/obj/ssync_receiver.py +145 -85
- swift/obj/ssync_sender.py +113 -54
- swift/obj/updater.py +652 -139
- swift/obj/watchers/__init__.py +0 -0
- swift/obj/watchers/dark_data.py +213 -0
- swift/proxy/controllers/account.py +11 -11
- swift/proxy/controllers/base.py +848 -604
- swift/proxy/controllers/container.py +433 -92
- swift/proxy/controllers/info.py +3 -2
- swift/proxy/controllers/obj.py +1000 -489
- swift/proxy/server.py +185 -112
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/AUTHORS +58 -11
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/METADATA +51 -56
- swift-2.35.0.dist-info/RECORD +201 -0
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
- swift-2.35.0.dist-info/pbr.json +1 -0
- swift/locale/de/LC_MESSAGES/swift.po +0 -1216
- swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
- swift/locale/es/LC_MESSAGES/swift.po +0 -1085
- swift/locale/fr/LC_MESSAGES/swift.po +0 -909
- swift/locale/it/LC_MESSAGES/swift.po +0 -894
- swift/locale/ja/LC_MESSAGES/swift.po +0 -965
- swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
- swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
- swift/locale/ru/LC_MESSAGES/swift.po +0 -891
- swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
- swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
- swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
- swift-2.23.3.data/scripts/swift-account-auditor +0 -23
- swift-2.23.3.data/scripts/swift-account-info +0 -51
- swift-2.23.3.data/scripts/swift-account-reaper +0 -23
- swift-2.23.3.data/scripts/swift-account-replicator +0 -34
- swift-2.23.3.data/scripts/swift-account-server +0 -23
- swift-2.23.3.data/scripts/swift-container-auditor +0 -23
- swift-2.23.3.data/scripts/swift-container-info +0 -55
- swift-2.23.3.data/scripts/swift-container-reconciler +0 -21
- swift-2.23.3.data/scripts/swift-container-replicator +0 -34
- swift-2.23.3.data/scripts/swift-container-sharder +0 -37
- swift-2.23.3.data/scripts/swift-container-sync +0 -23
- swift-2.23.3.data/scripts/swift-container-updater +0 -23
- swift-2.23.3.data/scripts/swift-dispersion-report +0 -24
- swift-2.23.3.data/scripts/swift-form-signature +0 -20
- swift-2.23.3.data/scripts/swift-init +0 -119
- swift-2.23.3.data/scripts/swift-object-auditor +0 -29
- swift-2.23.3.data/scripts/swift-object-expirer +0 -33
- swift-2.23.3.data/scripts/swift-object-info +0 -60
- swift-2.23.3.data/scripts/swift-object-reconstructor +0 -33
- swift-2.23.3.data/scripts/swift-object-relinker +0 -41
- swift-2.23.3.data/scripts/swift-object-replicator +0 -37
- swift-2.23.3.data/scripts/swift-object-server +0 -27
- swift-2.23.3.data/scripts/swift-object-updater +0 -23
- swift-2.23.3.data/scripts/swift-proxy-server +0 -23
- swift-2.23.3.data/scripts/swift-recon +0 -24
- swift-2.23.3.data/scripts/swift-ring-builder +0 -24
- swift-2.23.3.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.23.3.data/scripts/swift-ring-composer +0 -22
- swift-2.23.3.dist-info/RECORD +0 -220
- swift-2.23.3.dist-info/pbr.json +0 -1
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/LICENSE +0 -0
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
swift/common/middleware/bulk.py
CHANGED
@@ -195,9 +195,8 @@ payload sent to the proxy (the list of objects/containers to be deleted).
|
|
195
195
|
"""
|
196
196
|
|
197
197
|
import json
|
198
|
-
import six
|
199
198
|
import tarfile
|
200
|
-
from xml.sax import
|
199
|
+
from xml.sax.saxutils import escape # nosec B406
|
201
200
|
from time import time
|
202
201
|
from eventlet import sleep
|
203
202
|
import zlib
|
@@ -206,8 +205,8 @@ from swift.common.swob import Request, HTTPBadGateway, \
|
|
206
205
|
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \
|
207
206
|
HTTPLengthRequired, HTTPException, HTTPServerError, wsgify, \
|
208
207
|
bytes_to_wsgi, str_to_wsgi, wsgi_unquote, wsgi_quote, wsgi_to_str
|
209
|
-
from swift.common.utils import get_logger,
|
210
|
-
|
208
|
+
from swift.common.utils import get_logger, StreamingPile
|
209
|
+
from swift.common.registry import register_swift_info
|
211
210
|
from swift.common import constraints
|
212
211
|
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
|
213
212
|
from swift.common.request_helpers import is_user_meta
|
@@ -247,18 +246,16 @@ def get_response_body(data_format, data_dict, error_list, root_tag):
|
|
247
246
|
xml_key = key.replace(' ', '_').lower()
|
248
247
|
output.extend([
|
249
248
|
'<', xml_key, '>',
|
250
|
-
|
249
|
+
escape(str(data_dict[key])),
|
251
250
|
'</', xml_key, '>\n',
|
252
251
|
])
|
253
252
|
output.append('<errors>\n')
|
254
253
|
for name, status in error_list:
|
255
254
|
output.extend([
|
256
|
-
'<object><name>',
|
257
|
-
|
255
|
+
'<object><name>', escape(name), '</name><status>',
|
256
|
+
escape(status), '</status></object>\n',
|
258
257
|
])
|
259
258
|
output.extend(['</errors>\n</', root_tag, '>\n'])
|
260
|
-
if six.PY2:
|
261
|
-
return ''.join(output)
|
262
259
|
return ''.join(output).encode('utf-8')
|
263
260
|
|
264
261
|
output = []
|
@@ -268,8 +265,6 @@ def get_response_body(data_format, data_dict, error_list, root_tag):
|
|
268
265
|
output.extend(
|
269
266
|
'%s, %s\n' % (name, status)
|
270
267
|
for name, status in error_list)
|
271
|
-
if six.PY2:
|
272
|
-
return ''.join(output)
|
273
268
|
return ''.join(output).encode('utf-8')
|
274
269
|
|
275
270
|
|
@@ -279,13 +274,9 @@ def pax_key_to_swift_header(pax_key):
|
|
279
274
|
return "Content-Type"
|
280
275
|
elif pax_key.startswith(u"SCHILY.xattr.user.meta."):
|
281
276
|
useful_part = pax_key[len(u"SCHILY.xattr.user.meta."):]
|
282
|
-
if six.PY2:
|
283
|
-
return "X-Object-Meta-" + useful_part.encode("utf-8")
|
284
277
|
return str_to_wsgi("X-Object-Meta-" + useful_part)
|
285
278
|
elif pax_key.startswith(u"LIBARCHIVE.xattr.user.meta."):
|
286
279
|
useful_part = pax_key[len(u"LIBARCHIVE.xattr.user.meta."):]
|
287
|
-
if six.PY2:
|
288
|
-
return "X-Object-Meta-" + useful_part.encode("utf-8")
|
289
280
|
return str_to_wsgi("X-Object-Meta-" + useful_part)
|
290
281
|
else:
|
291
282
|
# You can get things like atime/mtime/ctime or filesystem ACLs in
|
@@ -357,15 +348,12 @@ class Bulk(object):
|
|
357
348
|
while data_remaining:
|
358
349
|
if b'\n' in line:
|
359
350
|
obj_to_delete, line = line.split(b'\n', 1)
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
# input like b'\xe2%98\x83'
|
367
|
-
obj_to_delete = wsgi_to_str(wsgi_unquote(
|
368
|
-
bytes_to_wsgi(obj_to_delete.strip())))
|
351
|
+
# yeah, all this chaining is pretty terrible...
|
352
|
+
# but it gets even worse trying to use UTF-8 and
|
353
|
+
# errors='surrogateescape' when dealing with terrible
|
354
|
+
# input like b'\xe2%98\x83'
|
355
|
+
obj_to_delete = wsgi_to_str(wsgi_unquote(
|
356
|
+
bytes_to_wsgi(obj_to_delete.strip())))
|
369
357
|
objs_to_delete.append({'name': obj_to_delete})
|
370
358
|
else:
|
371
359
|
data = req.body_file.read(self.max_path_length)
|
@@ -373,11 +361,8 @@ class Bulk(object):
|
|
373
361
|
line += data
|
374
362
|
else:
|
375
363
|
data_remaining = False
|
376
|
-
|
377
|
-
|
378
|
-
else:
|
379
|
-
obj_to_delete = wsgi_to_str(wsgi_unquote(
|
380
|
-
bytes_to_wsgi(line.strip())))
|
364
|
+
obj_to_delete = wsgi_to_str(wsgi_unquote(
|
365
|
+
bytes_to_wsgi(line.strip())))
|
381
366
|
if obj_to_delete:
|
382
367
|
objs_to_delete.append({'name': obj_to_delete})
|
383
368
|
if len(objs_to_delete) > self.max_deletes_per_request:
|
@@ -457,7 +442,8 @@ class Bulk(object):
|
|
457
442
|
failed_files.append([wsgi_quote(str_to_wsgi(obj_name)),
|
458
443
|
HTTPPreconditionFailed().status])
|
459
444
|
continue
|
460
|
-
yield (obj_name, delete_path
|
445
|
+
yield (obj_name, delete_path,
|
446
|
+
obj_to_delete.get('version_id'))
|
461
447
|
|
462
448
|
def objs_then_containers(objs_to_delete):
|
463
449
|
# process all objects first
|
@@ -467,13 +453,17 @@ class Bulk(object):
|
|
467
453
|
yield delete_filter(lambda name: '/' not in name.strip('/'),
|
468
454
|
objs_to_delete)
|
469
455
|
|
470
|
-
def do_delete(obj_name, delete_path):
|
456
|
+
def do_delete(obj_name, delete_path, version_id):
|
471
457
|
delete_obj_req = make_subrequest(
|
472
458
|
req.environ, method='DELETE',
|
473
459
|
path=wsgi_quote(str_to_wsgi(delete_path)),
|
474
460
|
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
|
475
461
|
body='', agent='%(orig)s ' + user_agent,
|
476
462
|
swift_source=swift_source)
|
463
|
+
if version_id is None:
|
464
|
+
delete_obj_req.params = {}
|
465
|
+
else:
|
466
|
+
delete_obj_req.params = {'version-id': version_id}
|
477
467
|
return (delete_obj_req.get_response(self.app), obj_name, 0)
|
478
468
|
|
479
469
|
with StreamingPile(self.delete_concurrency) as pile:
|
@@ -572,9 +562,7 @@ class Bulk(object):
|
|
572
562
|
len(failed_files) >= self.max_failed_extractions:
|
573
563
|
break
|
574
564
|
if tar_info.isfile():
|
575
|
-
obj_path = tar_info.name
|
576
|
-
if not six.PY2:
|
577
|
-
obj_path = obj_path.encode('utf-8', 'surrogateescape')
|
565
|
+
obj_path = tar_info.name.encode('utf-8', 'surrogateescape')
|
578
566
|
obj_path = bytes_to_wsgi(obj_path)
|
579
567
|
if obj_path.startswith('./'):
|
580
568
|
obj_path = obj_path[2:]
|
@@ -13,8 +13,6 @@
|
|
13
13
|
# See the License for the specific language governing permissions and
|
14
14
|
# limitations under the License.
|
15
15
|
|
16
|
-
from swift import gettext_ as _
|
17
|
-
|
18
16
|
from swift.common.swob import Request, HTTPServerError
|
19
17
|
from swift.common.utils import get_logger, generate_trans_id, close_if_possible
|
20
18
|
from swift.common.wsgi import WSGIContext
|
@@ -74,7 +72,7 @@ class CatchErrorsContext(WSGIContext):
|
|
74
72
|
# catch any errors in the pipeline
|
75
73
|
resp = self._app_call(env)
|
76
74
|
except: # noqa
|
77
|
-
self.logger.exception(
|
75
|
+
self.logger.exception('Error: An error occurred')
|
78
76
|
resp = HTTPServerError(request=Request(env),
|
79
77
|
body=b'An error occurred',
|
80
78
|
content_type='text/plain')
|
@@ -27,10 +27,6 @@ maximum lookup depth. If a match is found, the environment's Host header is
|
|
27
27
|
rewritten and the request is passed further down the WSGI chain.
|
28
28
|
"""
|
29
29
|
|
30
|
-
import six
|
31
|
-
|
32
|
-
from swift import gettext_ as _
|
33
|
-
|
34
30
|
try:
|
35
31
|
import dns.resolver
|
36
32
|
import dns.exception
|
@@ -44,7 +40,8 @@ from swift.common.middleware import RewriteContext
|
|
44
40
|
from swift.common.swob import Request, HTTPBadRequest, \
|
45
41
|
str_to_wsgi, wsgi_to_str
|
46
42
|
from swift.common.utils import cache_from_env, get_logger, is_valid_ip, \
|
47
|
-
list_from_csv, parse_socket_string
|
43
|
+
list_from_csv, parse_socket_string
|
44
|
+
from swift.common.registry import register_swift_info
|
48
45
|
|
49
46
|
|
50
47
|
def lookup_cname(domain, resolver): # pragma: no cover
|
@@ -59,7 +56,7 @@ def lookup_cname(domain, resolver): # pragma: no cover
|
|
59
56
|
try:
|
60
57
|
answer = resolver.query(domain, 'CNAME').rrset
|
61
58
|
ttl = answer.ttl
|
62
|
-
result = answer.items[0].to_text()
|
59
|
+
result = list(answer.items)[0].to_text()
|
63
60
|
result = result.rstrip('.')
|
64
61
|
return ttl, result
|
65
62
|
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
|
@@ -150,8 +147,6 @@ class CNAMELookupMiddleware(object):
|
|
150
147
|
if self.memcache:
|
151
148
|
memcache_key = ''.join(['cname-', a_domain])
|
152
149
|
found_domain = self.memcache.get(memcache_key)
|
153
|
-
if six.PY2 and found_domain:
|
154
|
-
found_domain = found_domain.encode('utf-8')
|
155
150
|
if found_domain is None:
|
156
151
|
ttl, found_domain = lookup_cname(a_domain, self.resolver)
|
157
152
|
if self.memcache and ttl > 0:
|
@@ -166,7 +161,7 @@ class CNAMELookupMiddleware(object):
|
|
166
161
|
elif self._domain_endswith_in_storage_domain(found_domain):
|
167
162
|
# Found it!
|
168
163
|
self.logger.info(
|
169
|
-
|
164
|
+
'Mapped %(given_domain)s to %(found_domain)s',
|
170
165
|
{'given_domain': given_domain,
|
171
166
|
'found_domain': found_domain})
|
172
167
|
if port:
|
@@ -179,8 +174,8 @@ class CNAMELookupMiddleware(object):
|
|
179
174
|
else:
|
180
175
|
# try one more deep in the chain
|
181
176
|
self.logger.debug(
|
182
|
-
|
183
|
-
|
177
|
+
'Following CNAME chain for '
|
178
|
+
'%(given_domain)s to %(found_domain)s',
|
184
179
|
{'given_domain': given_domain,
|
185
180
|
'found_domain': found_domain})
|
186
181
|
a_domain = found_domain
|
@@ -54,7 +54,7 @@ For example::
|
|
54
54
|
from swift.common.http import is_success
|
55
55
|
from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \
|
56
56
|
wsgify
|
57
|
-
from swift.common.
|
57
|
+
from swift.common.registry import register_swift_info
|
58
58
|
from swift.proxy.controllers.base import get_container_info
|
59
59
|
|
60
60
|
|
@@ -15,11 +15,14 @@
|
|
15
15
|
|
16
16
|
import os
|
17
17
|
|
18
|
+
from swift.common.constraints import valid_api_version
|
18
19
|
from swift.common.container_sync_realms import ContainerSyncRealms
|
20
|
+
from swift.common.request_helpers import append_log_info
|
19
21
|
from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify
|
20
22
|
from swift.common.utils import (
|
21
|
-
config_true_value, get_logger,
|
23
|
+
config_true_value, get_logger, streq_const_time)
|
22
24
|
from swift.proxy.controllers.base import get_container_info
|
25
|
+
from swift.common.registry import register_swift_info
|
23
26
|
|
24
27
|
|
25
28
|
class ContainerSync(object):
|
@@ -67,8 +70,35 @@ class ContainerSync(object):
|
|
67
70
|
|
68
71
|
@wsgify
|
69
72
|
def __call__(self, req):
|
73
|
+
if req.path == '/info':
|
74
|
+
# Ensure /info requests get the freshest results
|
75
|
+
self.register_info()
|
76
|
+
return self.app
|
77
|
+
|
78
|
+
try:
|
79
|
+
(version, acc, cont, obj) = req.split_path(3, 4, True)
|
80
|
+
bad_path = False
|
81
|
+
except ValueError:
|
82
|
+
bad_path = True
|
83
|
+
|
84
|
+
# use of bad_path bool is to avoid recursive tracebacks
|
85
|
+
if bad_path or not valid_api_version(version):
|
86
|
+
return self.app
|
87
|
+
|
88
|
+
# validate container-sync metdata update
|
89
|
+
info = get_container_info(
|
90
|
+
req.environ, self.app, swift_source='CS')
|
91
|
+
sync_to = req.headers.get('x-container-sync-to')
|
92
|
+
if req.method in ('PUT', 'POST') and cont and not obj:
|
93
|
+
versions_cont = info.get(
|
94
|
+
'sysmeta', {}).get('versions-container')
|
95
|
+
if sync_to and versions_cont:
|
96
|
+
raise HTTPBadRequest(
|
97
|
+
'Cannot configure container sync on a container '
|
98
|
+
'with object versioning configured.',
|
99
|
+
request=req)
|
100
|
+
|
70
101
|
if not self.allow_full_urls:
|
71
|
-
sync_to = req.headers.get('x-container-sync-to')
|
72
102
|
if sync_to and not sync_to.startswith('//'):
|
73
103
|
raise HTTPBadRequest(
|
74
104
|
body='Full URLs are not allowed for X-Container-Sync-To '
|
@@ -80,22 +110,17 @@ class ContainerSync(object):
|
|
80
110
|
valid = False
|
81
111
|
auth = auth.split()
|
82
112
|
if len(auth) != 3:
|
83
|
-
req.environ
|
84
|
-
'cs:not-3-args')
|
113
|
+
append_log_info(req.environ, 'cs:not-3-args')
|
85
114
|
else:
|
86
115
|
realm, nonce, sig = auth
|
87
116
|
realm_key = self.realms_conf.key(realm)
|
88
117
|
realm_key2 = self.realms_conf.key2(realm)
|
89
118
|
if not realm_key:
|
90
|
-
req.environ
|
91
|
-
'cs:no-local-realm-key')
|
119
|
+
append_log_info(req.environ, 'cs:no-local-realm-key')
|
92
120
|
else:
|
93
|
-
info = get_container_info(
|
94
|
-
req.environ, self.app, swift_source='CS')
|
95
121
|
user_key = info.get('sync_key')
|
96
122
|
if not user_key:
|
97
|
-
req.environ
|
98
|
-
'cs:no-local-user-key')
|
123
|
+
append_log_info(req.environ, 'cs:no-local-user-key')
|
99
124
|
else:
|
100
125
|
# x-timestamp headers get shunted by gatekeeper
|
101
126
|
if 'x-backend-inbound-x-timestamp' in req.headers:
|
@@ -112,11 +137,9 @@ class ContainerSync(object):
|
|
112
137
|
realm_key2, user_key) if realm_key2 else expected
|
113
138
|
if not streq_const_time(sig, expected) and \
|
114
139
|
not streq_const_time(sig, expected2):
|
115
|
-
req.environ
|
116
|
-
'swift.log_info', []).append('cs:invalid-sig')
|
140
|
+
append_log_info(req.environ, 'cs:invalid-sig')
|
117
141
|
else:
|
118
|
-
req.environ
|
119
|
-
'swift.log_info', []).append('cs:valid')
|
142
|
+
append_log_info(req.environ, 'cs:valid')
|
120
143
|
valid = True
|
121
144
|
if not valid:
|
122
145
|
exc = HTTPUnauthorized(
|
@@ -134,10 +157,9 @@ class ContainerSync(object):
|
|
134
157
|
# syntax and might be synced before its segments, so stop SLO
|
135
158
|
# middleware from performing the usual manifest validation.
|
136
159
|
req.environ['swift.slo_override'] = True
|
160
|
+
# Similar arguments for static symlinks
|
161
|
+
req.environ['swift.symlink_override'] = True
|
137
162
|
|
138
|
-
if req.path == '/info':
|
139
|
-
# Ensure /info requests get the freshest results
|
140
|
-
self.register_info()
|
141
163
|
return self.app
|
142
164
|
|
143
165
|
|
swift/common/middleware/copy.py
CHANGED
@@ -319,6 +319,9 @@ class ServerSideCopyMiddleware(object):
|
|
319
319
|
if 'last-modified' in source_resp.headers:
|
320
320
|
resp_headers['X-Copied-From-Last-Modified'] = \
|
321
321
|
source_resp.headers['last-modified']
|
322
|
+
if 'X-Object-Version-Id' in source_resp.headers:
|
323
|
+
resp_headers['X-Copied-From-Version-Id'] = \
|
324
|
+
source_resp.headers['X-Object-Version-Id']
|
322
325
|
# Existing sys and user meta of source object is added to response
|
323
326
|
# headers in addition to the new ones.
|
324
327
|
_copy_headers(sink_req.headers, resp_headers)
|
@@ -374,6 +377,8 @@ class ServerSideCopyMiddleware(object):
|
|
374
377
|
sink_req.headers.update(req.headers)
|
375
378
|
|
376
379
|
params = sink_req.params
|
380
|
+
params_updated = False
|
381
|
+
|
377
382
|
if params.get('multipart-manifest') == 'get':
|
378
383
|
if 'X-Static-Large-Object' in source_resp.headers:
|
379
384
|
params['multipart-manifest'] = 'put'
|
@@ -381,6 +386,13 @@ class ServerSideCopyMiddleware(object):
|
|
381
386
|
del params['multipart-manifest']
|
382
387
|
sink_req.headers['X-Object-Manifest'] = \
|
383
388
|
source_resp.headers['X-Object-Manifest']
|
389
|
+
params_updated = True
|
390
|
+
|
391
|
+
if 'version-id' in params:
|
392
|
+
del params['version-id']
|
393
|
+
params_updated = True
|
394
|
+
|
395
|
+
if params_updated:
|
384
396
|
sink_req.params = params
|
385
397
|
|
386
398
|
# Set swift.source, data source, content length and etag
|
@@ -14,7 +14,7 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
|
16
16
|
from swift.common.swob import Request, Response
|
17
|
-
from swift.common.
|
17
|
+
from swift.common.registry import register_swift_info
|
18
18
|
|
19
19
|
|
20
20
|
class CrossDomainMiddleware(object):
|
@@ -23,20 +23,24 @@ class CrossDomainMiddleware(object):
|
|
23
23
|
Cross domain middleware used to respond to requests for cross domain
|
24
24
|
policy information.
|
25
25
|
|
26
|
-
If the path is
|
27
|
-
policy document. This allows web pages hosted elsewhere to use
|
28
|
-
side technologies such as Flash, Java and Silverlight to interact
|
26
|
+
If the path is ``/crossdomain.xml`` it will respond with an xml cross
|
27
|
+
domain policy document. This allows web pages hosted elsewhere to use
|
28
|
+
client side technologies such as Flash, Java and Silverlight to interact
|
29
29
|
with the Swift API.
|
30
30
|
|
31
31
|
To enable this middleware, add it to the pipeline in your proxy-server.conf
|
32
32
|
file. It should be added before any authentication (e.g., tempauth or
|
33
33
|
keystone) middleware. In this example ellipsis (...) indicate other
|
34
|
-
middleware you may have chosen to use
|
34
|
+
middleware you may have chosen to use:
|
35
|
+
|
36
|
+
.. code:: cfg
|
35
37
|
|
36
38
|
[pipeline:main]
|
37
39
|
pipeline = ... crossdomain ... authtoken ... proxy-server
|
38
40
|
|
39
|
-
And add a filter section, such as
|
41
|
+
And add a filter section, such as:
|
42
|
+
|
43
|
+
.. code:: cfg
|
40
44
|
|
41
45
|
[filter:crossdomain]
|
42
46
|
use = egg:swift#crossdomain
|
@@ -45,13 +49,22 @@ class CrossDomainMiddleware(object):
|
|
45
49
|
|
46
50
|
For continuation lines, put some whitespace before the continuation
|
47
51
|
text. Ensure you put a completely blank line to terminate the
|
48
|
-
cross_domain_policy value.
|
52
|
+
``cross_domain_policy`` value.
|
53
|
+
|
54
|
+
The ``cross_domain_policy`` name/value is optional. If omitted, the policy
|
55
|
+
defaults as if you had specified:
|
49
56
|
|
50
|
-
|
51
|
-
defaults as if you had specified::
|
57
|
+
.. code:: cfg
|
52
58
|
|
53
59
|
cross_domain_policy = <allow-access-from domain="*" secure="false" />
|
54
60
|
|
61
|
+
.. note::
|
62
|
+
|
63
|
+
The default policy is very permissive; this is appropriate
|
64
|
+
for most public cloud deployments, but may not be appropriate
|
65
|
+
for all deployments. See also:
|
66
|
+
`CWE-942 <https://cwe.mitre.org/data/definitions/942.html>`__
|
67
|
+
|
55
68
|
|
56
69
|
"""
|
57
70
|
|
@@ -20,7 +20,8 @@ instance of an :class:`~swift.common.middleware.crypto.encrypter.Encrypter`.
|
|
20
20
|
from swift.common.middleware.crypto.decrypter import Decrypter
|
21
21
|
from swift.common.middleware.crypto.encrypter import Encrypter
|
22
22
|
|
23
|
-
from swift.common.utils import config_true_value
|
23
|
+
from swift.common.utils import config_true_value
|
24
|
+
from swift.common.registry import register_swift_info
|
24
25
|
|
25
26
|
|
26
27
|
def filter_factory(global_conf, **local_conf):
|
@@ -14,21 +14,17 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
import base64
|
16
16
|
import binascii
|
17
|
-
import collections
|
18
17
|
import json
|
19
18
|
import os
|
20
19
|
|
21
20
|
from cryptography.hazmat.backends import default_backend
|
22
21
|
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
23
|
-
import
|
24
|
-
from six.moves.urllib import parse as urlparse
|
22
|
+
import urllib.parse
|
25
23
|
|
26
|
-
from swift import gettext_ as _
|
27
24
|
from swift.common.exceptions import EncryptionException, UnknownSecretIdError
|
28
25
|
from swift.common.swob import HTTPInternalServerError
|
29
|
-
from swift.common.utils import get_logger
|
26
|
+
from swift.common.utils import get_logger, parse_header
|
30
27
|
from swift.common.wsgi import WSGIContext
|
31
|
-
from cgi import parse_header
|
32
28
|
|
33
29
|
CRYPTO_KEY_CALLBACK = 'swift.callback.fetch_crypto_keys'
|
34
30
|
|
@@ -161,7 +157,7 @@ class CryptoWSGIContext(WSGIContext):
|
|
161
157
|
try:
|
162
158
|
fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
|
163
159
|
except KeyError:
|
164
|
-
self.logger.exception(
|
160
|
+
self.logger.exception('ERROR get_keys() missing callback')
|
165
161
|
raise HTTPInternalServerError(
|
166
162
|
"Unable to retrieve encryption keys.")
|
167
163
|
|
@@ -182,12 +178,12 @@ class CryptoWSGIContext(WSGIContext):
|
|
182
178
|
self.crypto.check_key(key)
|
183
179
|
continue
|
184
180
|
except KeyError:
|
185
|
-
self.logger.exception(
|
181
|
+
self.logger.exception("Missing key for %r", name)
|
186
182
|
except TypeError:
|
187
|
-
self.logger.exception(
|
183
|
+
self.logger.exception("Did not get a keys dict")
|
188
184
|
except ValueError as e:
|
189
185
|
# don't include the key in any messages!
|
190
|
-
self.logger.exception(
|
186
|
+
self.logger.exception("Bad key for %(name)r: %(err)s",
|
191
187
|
{'name': name, 'err': e})
|
192
188
|
raise HTTPInternalServerError(
|
193
189
|
"Unable to retrieve encryption keys.")
|
@@ -227,7 +223,7 @@ def dump_crypto_meta(crypto_meta):
|
|
227
223
|
for name, value in crypto_meta.items()}
|
228
224
|
|
229
225
|
# use sort_keys=True to make serialized form predictable for testing
|
230
|
-
return
|
226
|
+
return urllib.parse.quote_plus(
|
231
227
|
json.dumps(b64_encode_meta(crypto_meta), sort_keys=True))
|
232
228
|
|
233
229
|
|
@@ -253,14 +249,14 @@ def load_crypto_meta(value, b64decode=True):
|
|
253
249
|
str(name): (
|
254
250
|
base64.b64decode(val) if name in ('iv', 'key') and b64decode
|
255
251
|
else b64_decode_meta(val) if isinstance(val, dict)
|
256
|
-
else val
|
252
|
+
else val)
|
257
253
|
for name, val in crypto_meta.items()}
|
258
254
|
|
259
255
|
try:
|
260
|
-
if not isinstance(value,
|
256
|
+
if not isinstance(value, str):
|
261
257
|
raise ValueError('crypto meta not a string')
|
262
|
-
val = json.loads(
|
263
|
-
if not isinstance(val,
|
258
|
+
val = json.loads(urllib.parse.unquote_plus(value))
|
259
|
+
if not isinstance(val, dict):
|
264
260
|
raise ValueError('crypto meta not a Mapping')
|
265
261
|
return b64_decode_meta(val)
|
266
262
|
except (KeyError, ValueError, TypeError) as err:
|
@@ -16,7 +16,6 @@
|
|
16
16
|
import base64
|
17
17
|
import json
|
18
18
|
|
19
|
-
from swift import gettext_ as _
|
20
19
|
from swift.common.header_key_dict import HeaderKeyDict
|
21
20
|
from swift.common.http import is_success
|
22
21
|
from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
|
@@ -77,10 +76,10 @@ class BaseDecrypterContext(CryptoWSGIContext):
|
|
77
76
|
crypto_meta['body_key'])
|
78
77
|
except KeyError as err:
|
79
78
|
self.logger.error(
|
80
|
-
|
79
|
+
'Error decrypting %(resp_type)s: Missing %(key)s',
|
81
80
|
{'resp_type': self.server_type, 'key': err})
|
82
81
|
except ValueError as err:
|
83
|
-
self.logger.error(
|
82
|
+
self.logger.error('Error decrypting %(resp_type)s: %(reason)s',
|
84
83
|
{'resp_type': self.server_type, 'reason': err})
|
85
84
|
raise HTTPInternalServerError(
|
86
85
|
body='Error decrypting %s' % self.server_type,
|
@@ -92,7 +91,7 @@ class BaseDecrypterContext(CryptoWSGIContext):
|
|
92
91
|
the value itself, otherwise return the value unmodified.
|
93
92
|
|
94
93
|
A value should either be a string that does not contain the ';'
|
95
|
-
character or should be of the form
|
94
|
+
character or should be of the form::
|
96
95
|
|
97
96
|
<base64-encoded ciphertext>;swift_meta=<crypto meta>
|
98
97
|
|
@@ -178,7 +177,7 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
178
177
|
value, key, required, bytes_to_wsgi)
|
179
178
|
except EncryptionException as err:
|
180
179
|
self.logger.error(
|
181
|
-
|
180
|
+
"Error decrypting header %(header)s: %(error)s",
|
182
181
|
{'header': header, 'error': err})
|
183
182
|
raise HTTPInternalServerError(
|
184
183
|
body='Error decrypting header',
|
@@ -197,7 +196,7 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
197
196
|
result.append((new_prefix + short_name, decrypted_value))
|
198
197
|
return result
|
199
198
|
|
200
|
-
def decrypt_resp_headers(self, put_keys, post_keys):
|
199
|
+
def decrypt_resp_headers(self, put_keys, post_keys, update_cors_exposed):
|
201
200
|
"""
|
202
201
|
Find encrypted headers and replace with the decrypted versions.
|
203
202
|
|
@@ -236,11 +235,27 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
236
235
|
# that map to the same x-object-meta- header names i.e. decrypted
|
237
236
|
# headers win over unexpected, unencrypted headers.
|
238
237
|
if post_keys:
|
239
|
-
|
238
|
+
decrypted_meta = self.decrypt_user_metadata(post_keys)
|
239
|
+
mod_hdr_pairs.extend(decrypted_meta)
|
240
|
+
else:
|
241
|
+
decrypted_meta = []
|
240
242
|
|
241
243
|
mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
|
242
|
-
|
243
|
-
|
244
|
+
|
245
|
+
found_aceh = False
|
246
|
+
for header, value in self._response_headers:
|
247
|
+
lheader = header.lower()
|
248
|
+
if lheader in mod_hdr_names:
|
249
|
+
continue
|
250
|
+
if lheader == 'access-control-expose-headers':
|
251
|
+
found_aceh = True
|
252
|
+
mod_hdr_pairs.append((header, value + ', ' + ', '.join(
|
253
|
+
meta.lower() for meta, _data in decrypted_meta)))
|
254
|
+
else:
|
255
|
+
mod_hdr_pairs.append((header, value))
|
256
|
+
if update_cors_exposed and not found_aceh:
|
257
|
+
mod_hdr_pairs.append(('Access-Control-Expose-Headers', ', '.join(
|
258
|
+
meta.lower() for meta, _data in decrypted_meta)))
|
244
259
|
return mod_hdr_pairs
|
245
260
|
|
246
261
|
def multipart_response_iter(self, resp, boundary, body_key, crypto_meta):
|
@@ -297,7 +312,7 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
297
312
|
try:
|
298
313
|
crypto_meta = self.get_crypto_meta(header, check)
|
299
314
|
except EncryptionException as err:
|
300
|
-
self.logger.error(
|
315
|
+
self.logger.error('Error decrypting object: %s', err)
|
301
316
|
raise HTTPInternalServerError(
|
302
317
|
body='Error decrypting object', content_type='text/plain')
|
303
318
|
return crypto_meta
|
@@ -326,7 +341,9 @@ class DecrypterObjContext(BaseDecrypterContext):
|
|
326
341
|
self._response_exc_info)
|
327
342
|
return app_resp
|
328
343
|
|
329
|
-
mod_resp_headers = self.decrypt_resp_headers(
|
344
|
+
mod_resp_headers = self.decrypt_resp_headers(
|
345
|
+
put_keys, post_keys,
|
346
|
+
update_cors_exposed=bool(req.headers.get('origin')))
|
330
347
|
|
331
348
|
if put_crypto_meta and req.method == 'GET' and \
|
332
349
|
is_success(self._get_status_int()):
|