swift 2.33.0__py2.py3-none-any.whl → 2.34.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/account/auditor.py +11 -0
- swift/account/reaper.py +11 -1
- swift/account/replicator.py +22 -0
- swift/account/server.py +12 -1
- swift-2.33.0.data/scripts/swift-account-audit → swift/cli/account_audit.py +6 -2
- swift-2.33.0.data/scripts/swift-config → swift/cli/config.py +1 -1
- swift-2.33.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +6 -2
- swift-2.33.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +12 -3
- swift-2.33.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +6 -2
- swift/cli/info.py +103 -2
- swift-2.33.0.data/scripts/swift-oldies → swift/cli/oldies.py +6 -3
- swift-2.33.0.data/scripts/swift-orphans → swift/cli/orphans.py +7 -2
- swift/cli/recon_cron.py +5 -5
- swift-2.33.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +1 -1
- swift/cli/ringbuilder.py +24 -0
- swift/common/db.py +2 -1
- swift/common/db_auditor.py +2 -2
- swift/common/db_replicator.py +6 -0
- swift/common/exceptions.py +12 -0
- swift/common/manager.py +102 -0
- swift/common/memcached.py +6 -13
- swift/common/middleware/account_quotas.py +144 -43
- swift/common/middleware/backend_ratelimit.py +166 -24
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +3 -5
- swift/common/middleware/container_sync.py +6 -10
- swift/common/middleware/crypto/crypto_utils.py +4 -5
- swift/common/middleware/crypto/decrypter.py +4 -5
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/listing_formats.py +26 -38
- swift/common/middleware/proxy_logging.py +22 -16
- swift/common/middleware/ratelimit.py +6 -7
- swift/common/middleware/recon.py +6 -7
- swift/common/middleware/s3api/acl_handlers.py +9 -0
- swift/common/middleware/s3api/controllers/multi_upload.py +1 -9
- swift/common/middleware/s3api/controllers/obj.py +20 -1
- swift/common/middleware/s3api/s3api.py +2 -0
- swift/common/middleware/s3api/s3request.py +171 -62
- swift/common/middleware/s3api/s3response.py +35 -6
- swift/common/middleware/s3api/s3token.py +2 -2
- swift/common/middleware/s3api/utils.py +1 -0
- swift/common/middleware/slo.py +153 -52
- swift/common/middleware/tempauth.py +6 -4
- swift/common/middleware/tempurl.py +2 -2
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +10 -11
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +6 -2
- swift/common/request_helpers.py +69 -0
- swift/common/statsd_client.py +207 -0
- swift/common/utils/__init__.py +97 -1635
- swift/common/utils/base.py +138 -0
- swift/common/utils/config.py +443 -0
- swift/common/utils/logs.py +999 -0
- swift/common/wsgi.py +11 -3
- swift/container/auditor.py +11 -0
- swift/container/backend.py +10 -10
- swift/container/reconciler.py +11 -2
- swift/container/replicator.py +22 -1
- swift/container/server.py +12 -1
- swift/container/sharder.py +36 -12
- swift/container/sync.py +11 -1
- swift/container/updater.py +11 -2
- swift/obj/auditor.py +18 -2
- swift/obj/diskfile.py +8 -6
- swift/obj/expirer.py +155 -36
- swift/obj/reconstructor.py +28 -4
- swift/obj/replicator.py +61 -22
- swift/obj/server.py +64 -36
- swift/obj/updater.py +11 -2
- swift/proxy/controllers/base.py +38 -22
- swift/proxy/controllers/obj.py +23 -26
- swift/proxy/server.py +15 -1
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/AUTHORS +11 -3
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/METADATA +34 -35
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/RECORD +82 -108
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/WHEEL +1 -1
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/entry_points.txt +38 -1
- swift-2.34.1.dist-info/pbr.json +1 -0
- swift-2.33.0.data/scripts/swift-account-auditor +0 -23
- swift-2.33.0.data/scripts/swift-account-info +0 -52
- swift-2.33.0.data/scripts/swift-account-reaper +0 -23
- swift-2.33.0.data/scripts/swift-account-replicator +0 -34
- swift-2.33.0.data/scripts/swift-account-server +0 -23
- swift-2.33.0.data/scripts/swift-container-auditor +0 -23
- swift-2.33.0.data/scripts/swift-container-info +0 -59
- swift-2.33.0.data/scripts/swift-container-reconciler +0 -21
- swift-2.33.0.data/scripts/swift-container-replicator +0 -34
- swift-2.33.0.data/scripts/swift-container-server +0 -23
- swift-2.33.0.data/scripts/swift-container-sharder +0 -37
- swift-2.33.0.data/scripts/swift-container-sync +0 -23
- swift-2.33.0.data/scripts/swift-container-updater +0 -23
- swift-2.33.0.data/scripts/swift-dispersion-report +0 -24
- swift-2.33.0.data/scripts/swift-form-signature +0 -20
- swift-2.33.0.data/scripts/swift-init +0 -119
- swift-2.33.0.data/scripts/swift-object-auditor +0 -29
- swift-2.33.0.data/scripts/swift-object-expirer +0 -33
- swift-2.33.0.data/scripts/swift-object-info +0 -60
- swift-2.33.0.data/scripts/swift-object-reconstructor +0 -33
- swift-2.33.0.data/scripts/swift-object-relinker +0 -23
- swift-2.33.0.data/scripts/swift-object-replicator +0 -37
- swift-2.33.0.data/scripts/swift-object-server +0 -27
- swift-2.33.0.data/scripts/swift-object-updater +0 -23
- swift-2.33.0.data/scripts/swift-proxy-server +0 -23
- swift-2.33.0.data/scripts/swift-recon +0 -24
- swift-2.33.0.data/scripts/swift-recon-cron +0 -24
- swift-2.33.0.data/scripts/swift-ring-builder +0 -37
- swift-2.33.0.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.33.0.data/scripts/swift-ring-composer +0 -22
- swift-2.33.0.dist-info/pbr.json +0 -1
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/LICENSE +0 -0
- {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/top_level.txt +0 -0
swift/obj/server.py
CHANGED
@@ -21,6 +21,7 @@ from six.moves.urllib.parse import unquote
|
|
21
21
|
import json
|
22
22
|
import os
|
23
23
|
import multiprocessing
|
24
|
+
import sys
|
24
25
|
import time
|
25
26
|
import traceback
|
26
27
|
import socket
|
@@ -34,7 +35,7 @@ from swift.common.utils import public, get_logger, \
|
|
34
35
|
get_expirer_container, parse_mime_headers, \
|
35
36
|
iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
|
36
37
|
config_auto_int_value, split_path, get_redirect_data, \
|
37
|
-
normalize_timestamp, md5
|
38
|
+
normalize_timestamp, md5, parse_options
|
38
39
|
from swift.common.bufferedhttp import http_connect
|
39
40
|
from swift.common.constraints import check_object_creation, \
|
40
41
|
valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
|
@@ -50,15 +51,18 @@ from swift.common.base_storage_server import BaseStorageServer
|
|
50
51
|
from swift.common.header_key_dict import HeaderKeyDict
|
51
52
|
from swift.common.request_helpers import get_name_and_placement, \
|
52
53
|
is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
|
53
|
-
resolve_etag_is_at_header, is_sys_meta, validate_internal_obj
|
54
|
+
resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \
|
55
|
+
is_backend_open_expired
|
54
56
|
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
|
55
57
|
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
|
56
58
|
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
|
57
59
|
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
|
58
60
|
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
|
59
61
|
HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
|
62
|
+
from swift.common.wsgi import run_wsgi
|
60
63
|
from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
|
61
|
-
from swift.obj.expirer import build_task_obj
|
64
|
+
from swift.obj.expirer import build_task_obj, embed_expirer_bytes_in_ctype, \
|
65
|
+
X_DELETE_TYPE
|
62
66
|
|
63
67
|
|
64
68
|
def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size):
|
@@ -268,7 +272,8 @@ class ObjectController(BaseStorageServer):
|
|
268
272
|
|
269
273
|
def async_update(self, op, account, container, obj, host, partition,
|
270
274
|
contdevice, headers_out, objdevice, policy,
|
271
|
-
logger_thread_locals=None, container_path=None
|
275
|
+
logger_thread_locals=None, container_path=None,
|
276
|
+
db_state=None):
|
272
277
|
"""
|
273
278
|
Sends or saves an async update.
|
274
279
|
|
@@ -290,6 +295,8 @@ class ObjectController(BaseStorageServer):
|
|
290
295
|
to which the update should be sent. If given this path will be used
|
291
296
|
instead of constructing a path from the ``account`` and
|
292
297
|
``container`` params.
|
298
|
+
:param db_state: The current database state of the container as
|
299
|
+
supplied to us by the proxy.
|
293
300
|
"""
|
294
301
|
if logger_thread_locals:
|
295
302
|
self.logger.thread_locals = logger_thread_locals
|
@@ -333,7 +340,7 @@ class ObjectController(BaseStorageServer):
|
|
333
340
|
'%(ip)s:%(port)s/%(dev)s (saving for async update later)',
|
334
341
|
{'ip': ip, 'port': port, 'dev': contdevice})
|
335
342
|
data = {'op': op, 'account': account, 'container': container,
|
336
|
-
'obj': obj, 'headers': headers_out}
|
343
|
+
'obj': obj, 'headers': headers_out, 'db_state': db_state}
|
337
344
|
if redirect_data:
|
338
345
|
self.logger.debug(
|
339
346
|
'Update to %(path)s redirected to %(redirect)s',
|
@@ -367,6 +374,7 @@ class ObjectController(BaseStorageServer):
|
|
367
374
|
contdevices = [d.strip() for d in
|
368
375
|
headers_in.get('X-Container-Device', '').split(',')]
|
369
376
|
contpartition = headers_in.get('X-Container-Partition', '')
|
377
|
+
contdbstate = headers_in.get('X-Container-Root-Db-State')
|
370
378
|
|
371
379
|
if len(conthosts) != len(contdevices):
|
372
380
|
# This shouldn't happen unless there's a bug in the proxy,
|
@@ -417,7 +425,7 @@ class ObjectController(BaseStorageServer):
|
|
417
425
|
conthost, contpartition, contdevice, headers_out,
|
418
426
|
objdevice, policy,
|
419
427
|
logger_thread_locals=self.logger.thread_locals,
|
420
|
-
container_path=contpath)
|
428
|
+
container_path=contpath, db_state=contdbstate)
|
421
429
|
update_greenthreads.append(gt)
|
422
430
|
# Wait a little bit to see if the container updates are successful.
|
423
431
|
# If we immediately return after firing off the greenthread above, then
|
@@ -436,7 +444,7 @@ class ObjectController(BaseStorageServer):
|
|
436
444
|
self.container_update_timeout, updates)
|
437
445
|
|
438
446
|
def delete_at_update(self, op, delete_at, account, container, obj,
|
439
|
-
request, objdevice, policy):
|
447
|
+
request, objdevice, policy, extra_headers=None):
|
440
448
|
"""
|
441
449
|
Update the expiring objects container when objects are updated.
|
442
450
|
|
@@ -448,6 +456,7 @@ class ObjectController(BaseStorageServer):
|
|
448
456
|
:param request: the original request driving the update
|
449
457
|
:param objdevice: device name that the object is in
|
450
458
|
:param policy: the BaseStoragePolicy instance (used for tmp dir)
|
459
|
+
:param extra_headers: dict of additional headers for the update
|
451
460
|
"""
|
452
461
|
if config_true_value(
|
453
462
|
request.headers.get('x-backend-replication', 'f')):
|
@@ -493,8 +502,10 @@ class ObjectController(BaseStorageServer):
|
|
493
502
|
if not updates:
|
494
503
|
updates = [(None, None)]
|
495
504
|
headers_out['x-size'] = '0'
|
496
|
-
headers_out['x-content-type'] =
|
505
|
+
headers_out['x-content-type'] = X_DELETE_TYPE
|
497
506
|
headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
|
507
|
+
if extra_headers:
|
508
|
+
headers_out.update(extra_headers)
|
498
509
|
else:
|
499
510
|
if not config_true_value(
|
500
511
|
request.headers.get(
|
@@ -619,6 +630,24 @@ class ObjectController(BaseStorageServer):
|
|
619
630
|
override = key.lower().replace(override_prefix, 'x-')
|
620
631
|
update_headers[override] = val
|
621
632
|
|
633
|
+
def _conditional_delete_at_update(self, request, device, account,
|
634
|
+
container, obj, policy, metadata,
|
635
|
+
orig_delete_at, new_delete_at):
|
636
|
+
if new_delete_at:
|
637
|
+
extra_headers = {
|
638
|
+
'x-content-type': embed_expirer_bytes_in_ctype(
|
639
|
+
X_DELETE_TYPE, metadata),
|
640
|
+
'x-content-type-timestamp':
|
641
|
+
metadata.get('X-Timestamp'),
|
642
|
+
}
|
643
|
+
self.delete_at_update(
|
644
|
+
'PUT', new_delete_at, account, container, obj, request,
|
645
|
+
device, policy, extra_headers)
|
646
|
+
if orig_delete_at and orig_delete_at != new_delete_at:
|
647
|
+
self.delete_at_update(
|
648
|
+
'DELETE', orig_delete_at, account, container, obj,
|
649
|
+
request, device, policy)
|
650
|
+
|
622
651
|
@public
|
623
652
|
@timing_stats()
|
624
653
|
def POST(self, request):
|
@@ -635,8 +664,7 @@ class ObjectController(BaseStorageServer):
|
|
635
664
|
try:
|
636
665
|
disk_file = self.get_diskfile(
|
637
666
|
device, partition, account, container, obj,
|
638
|
-
policy=policy, open_expired=
|
639
|
-
request.headers.get('x-backend-replication', 'false')),
|
667
|
+
policy=policy, open_expired=is_backend_open_expired(request),
|
640
668
|
next_part_power=next_part_power)
|
641
669
|
except DiskFileDeviceUnavailable:
|
642
670
|
return HTTPInsufficientStorage(drive=device, request=request)
|
@@ -675,15 +703,11 @@ class ObjectController(BaseStorageServer):
|
|
675
703
|
wsgi_to_bytes(header_key).title())
|
676
704
|
metadata[header_caps] = request.headers[header_key]
|
677
705
|
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
if orig_delete_at:
|
684
|
-
self.delete_at_update('DELETE', orig_delete_at, account,
|
685
|
-
container, obj, request, device,
|
686
|
-
policy)
|
706
|
+
disk_file_metadata = disk_file.get_datafile_metadata()
|
707
|
+
self._conditional_delete_at_update(
|
708
|
+
request, device, account, container, obj, policy,
|
709
|
+
disk_file_metadata, orig_delete_at, new_delete_at
|
710
|
+
)
|
687
711
|
else:
|
688
712
|
# preserve existing metadata, only content-type may be updated
|
689
713
|
metadata = dict(disk_file.get_metafile_metadata())
|
@@ -993,15 +1017,10 @@ class ObjectController(BaseStorageServer):
|
|
993
1017
|
orig_metadata, footers_metadata, metadata):
|
994
1018
|
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
|
995
1019
|
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
|
996
|
-
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
device, policy)
|
1001
|
-
if orig_delete_at:
|
1002
|
-
self.delete_at_update(
|
1003
|
-
'DELETE', orig_delete_at, account, container, obj,
|
1004
|
-
request, device, policy)
|
1020
|
+
|
1021
|
+
self._conditional_delete_at_update(request, device, account, container,
|
1022
|
+
obj, policy, metadata,
|
1023
|
+
orig_delete_at, new_delete_at)
|
1005
1024
|
|
1006
1025
|
update_headers = HeaderKeyDict({
|
1007
1026
|
'x-size': metadata['Content-Length'],
|
@@ -1074,8 +1093,7 @@ class ObjectController(BaseStorageServer):
|
|
1074
1093
|
disk_file = self.get_diskfile(
|
1075
1094
|
device, partition, account, container, obj,
|
1076
1095
|
policy=policy, frag_prefs=frag_prefs,
|
1077
|
-
open_expired=
|
1078
|
-
request.headers.get('x-backend-replication', 'false')))
|
1096
|
+
open_expired=is_backend_open_expired(request))
|
1079
1097
|
except DiskFileDeviceUnavailable:
|
1080
1098
|
return HTTPInsufficientStorage(drive=device, request=request)
|
1081
1099
|
try:
|
@@ -1157,8 +1175,7 @@ class ObjectController(BaseStorageServer):
|
|
1157
1175
|
disk_file = self.get_diskfile(
|
1158
1176
|
device, partition, account, container, obj,
|
1159
1177
|
policy=policy, frag_prefs=frag_prefs,
|
1160
|
-
open_expired=
|
1161
|
-
request.headers.get('x-backend-replication', 'false')))
|
1178
|
+
open_expired=is_backend_open_expired(request))
|
1162
1179
|
except DiskFileDeviceUnavailable:
|
1163
1180
|
return HTTPInsufficientStorage(drive=device, request=request)
|
1164
1181
|
try:
|
@@ -1264,10 +1281,10 @@ class ObjectController(BaseStorageServer):
|
|
1264
1281
|
else:
|
1265
1282
|
# differentiate success from no object at all
|
1266
1283
|
response_class = HTTPNoContent
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1284
|
+
self._conditional_delete_at_update(
|
1285
|
+
request, device, account, container, obj, policy, {},
|
1286
|
+
orig_delete_at, 0
|
1287
|
+
)
|
1271
1288
|
if orig_timestamp < req_timestamp:
|
1272
1289
|
try:
|
1273
1290
|
disk_file.delete(req_timestamp)
|
@@ -1443,3 +1460,14 @@ def app_factory(global_conf, **local_conf):
|
|
1443
1460
|
conf = global_conf.copy()
|
1444
1461
|
conf.update(local_conf)
|
1445
1462
|
return ObjectController(conf)
|
1463
|
+
|
1464
|
+
|
1465
|
+
def main():
|
1466
|
+
conf_file, options = parse_options(test_config=True)
|
1467
|
+
sys.exit(run_wsgi(conf_file, 'object-server',
|
1468
|
+
global_conf_callback=global_conf_callback,
|
1469
|
+
**options))
|
1470
|
+
|
1471
|
+
|
1472
|
+
if __name__ == '__main__':
|
1473
|
+
main()
|
swift/obj/updater.py
CHANGED
@@ -34,8 +34,8 @@ from swift.common.utils import get_logger, renamer, write_pickle, \
|
|
34
34
|
dump_recon_cache, config_true_value, RateLimitedIterator, split_path, \
|
35
35
|
eventlet_monkey_patch, get_redirect_data, ContextPool, hash_path, \
|
36
36
|
non_negative_float, config_positive_int_value, non_negative_int, \
|
37
|
-
EventletRateLimiter, node_to_string
|
38
|
-
from swift.common.daemon import Daemon
|
37
|
+
EventletRateLimiter, node_to_string, parse_options
|
38
|
+
from swift.common.daemon import Daemon, run_daemon
|
39
39
|
from swift.common.header_key_dict import HeaderKeyDict
|
40
40
|
from swift.common.storage_policy import split_policy_string, PolicyError
|
41
41
|
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
|
@@ -758,3 +758,12 @@ class ObjectUpdater(Daemon):
|
|
758
758
|
self.logger.timing('updater.timing.status.%s' % status,
|
759
759
|
elapsed * 1000)
|
760
760
|
return HTTP_INTERNAL_SERVER_ERROR, node['id'], redirect
|
761
|
+
|
762
|
+
|
763
|
+
def main():
|
764
|
+
conf_file, options = parse_options(once=True)
|
765
|
+
run_daemon(ObjectUpdater, conf_file, **options)
|
766
|
+
|
767
|
+
|
768
|
+
if __name__ == '__main__':
|
769
|
+
main()
|
swift/proxy/controllers/base.py
CHANGED
@@ -45,7 +45,7 @@ from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
|
|
45
45
|
public, split_path, list_from_csv, GreenthreadSafeIterator, \
|
46
46
|
GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
|
47
47
|
document_iters_to_http_response_body, cache_from_env, \
|
48
|
-
CooperativeIterator, NamespaceBoundList, Namespace
|
48
|
+
CooperativeIterator, NamespaceBoundList, Namespace, ClosingMapper
|
49
49
|
from swift.common.bufferedhttp import http_connect
|
50
50
|
from swift.common import constraints
|
51
51
|
from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
|
@@ -1107,6 +1107,18 @@ def is_good_source(status, server_type):
|
|
1107
1107
|
return is_success(status) or is_redirection(status)
|
1108
1108
|
|
1109
1109
|
|
1110
|
+
def is_useful_response(resp, node):
|
1111
|
+
if not resp:
|
1112
|
+
return False
|
1113
|
+
if ('handoff_index' in node
|
1114
|
+
and resp.status == 404
|
1115
|
+
and resp.headers.get('x-backend-timestamp') is None):
|
1116
|
+
# a 404 from a handoff are not considered authoritative unless they
|
1117
|
+
# have an x-backend-timestamp that indicates that there is a tombstone
|
1118
|
+
return False
|
1119
|
+
return True
|
1120
|
+
|
1121
|
+
|
1110
1122
|
class ByteCountEnforcer(object):
|
1111
1123
|
"""
|
1112
1124
|
Enforces that successive calls to file_like.read() give at least
|
@@ -1713,7 +1725,7 @@ class GetOrHeadHandler(GetterBase):
|
|
1713
1725
|
return response_part
|
1714
1726
|
|
1715
1727
|
return document_iters_to_http_response_body(
|
1716
|
-
(add_content_type
|
1728
|
+
ClosingMapper(add_content_type, parts_iter),
|
1717
1729
|
boundary, is_multipart, self.logger)
|
1718
1730
|
|
1719
1731
|
def get_working_response(self):
|
@@ -1956,7 +1968,7 @@ class Controller(object):
|
|
1956
1968
|
def generate_request_headers(self, orig_req=None, additional=None,
|
1957
1969
|
transfer=False):
|
1958
1970
|
"""
|
1959
|
-
Create a
|
1971
|
+
Create a dict of headers to be used in backend requests
|
1960
1972
|
|
1961
1973
|
:param orig_req: the original request sent by the client to the proxy
|
1962
1974
|
:param additional: additional headers to send to the backend
|
@@ -2088,14 +2100,14 @@ class Controller(object):
|
|
2088
2100
|
if (self.app.check_response(node, self.server_type, resp,
|
2089
2101
|
method, path)
|
2090
2102
|
and not is_informational(resp.status)):
|
2091
|
-
return resp
|
2092
|
-
resp.read()
|
2103
|
+
return resp, resp.read(), node
|
2093
2104
|
|
2094
2105
|
except (Exception, Timeout):
|
2095
2106
|
self.app.exception_occurred(
|
2096
2107
|
node, self.server_type,
|
2097
2108
|
'Trying to %(method)s %(path)s' %
|
2098
2109
|
{'method': method, 'path': path})
|
2110
|
+
return None, None, None
|
2099
2111
|
|
2100
2112
|
def make_requests(self, req, ring, part, method, path, headers,
|
2101
2113
|
query_string='', overrides=None, node_count=None,
|
@@ -2118,6 +2130,8 @@ class Controller(object):
|
|
2118
2130
|
the returned status of a request.
|
2119
2131
|
:param node_count: optional number of nodes to send request to.
|
2120
2132
|
:param node_iterator: optional node iterator.
|
2133
|
+
:param body: byte string to use as the request body.
|
2134
|
+
Try to keep it small.
|
2121
2135
|
:returns: a swob.Response object
|
2122
2136
|
"""
|
2123
2137
|
nodes = GreenthreadSafeIterator(node_iterator or NodeIter(
|
@@ -2128,25 +2142,25 @@ class Controller(object):
|
|
2128
2142
|
for head in headers:
|
2129
2143
|
pile.spawn(self._make_request, nodes, part, method, path,
|
2130
2144
|
head, query_string, body, self.logger.thread_locals)
|
2131
|
-
|
2145
|
+
results = []
|
2132
2146
|
statuses = []
|
2133
|
-
for resp in pile:
|
2134
|
-
if not resp:
|
2147
|
+
for resp, body, node in pile:
|
2148
|
+
if not is_useful_response(resp, node):
|
2135
2149
|
continue
|
2136
|
-
|
2137
|
-
statuses.append(resp
|
2150
|
+
results.append((resp.status, resp.reason, resp.getheaders(), body))
|
2151
|
+
statuses.append(resp.status)
|
2138
2152
|
if self.have_quorum(statuses, node_number):
|
2139
2153
|
break
|
2140
2154
|
# give any pending requests *some* chance to finish
|
2141
2155
|
finished_quickly = pile.waitall(self.app.post_quorum_timeout)
|
2142
|
-
for resp in finished_quickly:
|
2143
|
-
if not resp:
|
2156
|
+
for resp, body, node in finished_quickly:
|
2157
|
+
if not is_useful_response(resp, node):
|
2144
2158
|
continue
|
2145
|
-
|
2146
|
-
statuses.append(resp
|
2147
|
-
while len(
|
2148
|
-
|
2149
|
-
statuses, reasons, resp_headers, bodies = zip(*
|
2159
|
+
results.append((resp.status, resp.reason, resp.getheaders(), body))
|
2160
|
+
statuses.append(resp.status)
|
2161
|
+
while len(results) < node_number:
|
2162
|
+
results.append((HTTP_SERVICE_UNAVAILABLE, '', '', b''))
|
2163
|
+
statuses, reasons, resp_headers, bodies = zip(*results)
|
2150
2164
|
return self.best_response(req, statuses, reasons, bodies,
|
2151
2165
|
'%s %s' % (self.server_type, req.method),
|
2152
2166
|
overrides=overrides, headers=resp_headers)
|
@@ -2452,9 +2466,10 @@ class Controller(object):
|
|
2452
2466
|
|
2453
2467
|
def _parse_listing_response(self, req, response):
|
2454
2468
|
if not is_success(response.status_int):
|
2469
|
+
record_type = req.headers.get('X-Backend-Record-Type')
|
2455
2470
|
self.logger.warning(
|
2456
|
-
'Failed to get container listing from %s: %s',
|
2457
|
-
req.path_qs, response.status_int)
|
2471
|
+
'Failed to get container %s listing from %s: %s',
|
2472
|
+
record_type, req.path_qs, response.status_int)
|
2458
2473
|
return None
|
2459
2474
|
|
2460
2475
|
try:
|
@@ -2463,9 +2478,10 @@ class Controller(object):
|
|
2463
2478
|
raise ValueError('not a list')
|
2464
2479
|
return data
|
2465
2480
|
except ValueError as err:
|
2481
|
+
record_type = response.headers.get('X-Backend-Record-Type')
|
2466
2482
|
self.logger.error(
|
2467
|
-
'Problem with listing response from %s: %r',
|
2468
|
-
req.path_qs, err)
|
2483
|
+
'Problem with container %s listing response from %s: %r',
|
2484
|
+
record_type, req.path_qs, err)
|
2469
2485
|
return None
|
2470
2486
|
|
2471
2487
|
def _get_container_listing(self, req, account, container, headers=None,
|
@@ -2495,7 +2511,7 @@ class Controller(object):
|
|
2495
2511
|
self.logger.debug(
|
2496
2512
|
'Get listing from %s %s' % (subreq.path_qs, headers))
|
2497
2513
|
response = self.app.handle_request(subreq)
|
2498
|
-
data = self._parse_listing_response(
|
2514
|
+
data = self._parse_listing_response(subreq, response)
|
2499
2515
|
return data, response
|
2500
2516
|
|
2501
2517
|
def _parse_namespaces(self, req, listing, response):
|
swift/proxy/controllers/obj.py
CHANGED
@@ -77,7 +77,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
|
|
77
77
|
HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \
|
78
78
|
normalize_etag, str_to_wsgi
|
79
79
|
from swift.common.request_helpers import update_etag_is_at_header, \
|
80
|
-
resolve_etag_is_at_header, validate_internal_obj, get_ip_port
|
80
|
+
resolve_etag_is_at_header, validate_internal_obj, get_ip_port, \
|
81
|
+
is_open_expired, append_log_info
|
81
82
|
|
82
83
|
|
83
84
|
def check_content_type(req):
|
@@ -250,6 +251,8 @@ class BaseObjectController(Controller):
|
|
250
251
|
policy = POLICIES.get_by_index(policy_index)
|
251
252
|
obj_ring = self.app.get_object_ring(policy_index)
|
252
253
|
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
|
254
|
+
if is_open_expired(self.app, req):
|
255
|
+
req.headers['X-Backend-Open-Expired'] = 'true'
|
253
256
|
if 'swift.authorize' in req.environ:
|
254
257
|
aresp = req.environ['swift.authorize'](req)
|
255
258
|
if aresp:
|
@@ -388,9 +391,10 @@ class BaseObjectController(Controller):
|
|
388
391
|
if update_shard_ns:
|
389
392
|
partition, nodes = self.app.container_ring.get_nodes(
|
390
393
|
update_shard_ns.account, update_shard_ns.container)
|
391
|
-
return partition, nodes, update_shard_ns.name
|
394
|
+
return partition, nodes, update_shard_ns.name, db_state
|
392
395
|
|
393
|
-
return container_info['partition'], container_info['nodes'], None
|
396
|
+
return (container_info['partition'], container_info['nodes'], None,
|
397
|
+
db_state)
|
394
398
|
|
395
399
|
@public
|
396
400
|
@cors_validation
|
@@ -399,14 +403,14 @@ class BaseObjectController(Controller):
|
|
399
403
|
"""HTTP POST request handler."""
|
400
404
|
container_info = self.container_info(
|
401
405
|
self.account_name, self.container_name, req)
|
402
|
-
container_partition, container_nodes, container_path = \
|
403
|
-
self._get_update_target(req, container_info)
|
404
406
|
req.acl = container_info['write_acl']
|
407
|
+
if is_open_expired(self.app, req):
|
408
|
+
req.headers['X-Backend-Open-Expired'] = 'true'
|
405
409
|
if 'swift.authorize' in req.environ:
|
406
410
|
aresp = req.environ['swift.authorize'](req)
|
407
411
|
if aresp:
|
408
412
|
return aresp
|
409
|
-
if not
|
413
|
+
if not is_success(container_info.get('status')):
|
410
414
|
return HTTPNotFound(request=req)
|
411
415
|
error_response = check_metadata(req, 'object')
|
412
416
|
if error_response:
|
@@ -429,17 +433,17 @@ class BaseObjectController(Controller):
|
|
429
433
|
self.account_name, self.container_name, self.object_name)
|
430
434
|
|
431
435
|
headers = self._backend_requests(
|
432
|
-
req, len(nodes),
|
433
|
-
|
434
|
-
container_path=container_path)
|
436
|
+
req, len(nodes), container_info, delete_at_container,
|
437
|
+
delete_at_part, delete_at_nodes)
|
435
438
|
return self._post_object(req, obj_ring, partition, headers)
|
436
439
|
|
437
440
|
def _backend_requests(self, req, n_outgoing,
|
438
|
-
|
439
|
-
|
440
|
-
delete_at_nodes=None, container_path=None):
|
441
|
+
container_info, delete_at_container=None,
|
442
|
+
delete_at_partition=None, delete_at_nodes=None):
|
441
443
|
policy_index = req.headers['X-Backend-Storage-Policy-Index']
|
442
444
|
policy = POLICIES.get_by_index(policy_index)
|
445
|
+
container_partition, containers, container_path, db_state = \
|
446
|
+
self._get_update_target(req, container_info)
|
443
447
|
headers = [self.generate_request_headers(req, additional=req.headers)
|
444
448
|
for _junk in range(n_outgoing)]
|
445
449
|
|
@@ -452,6 +456,7 @@ class BaseObjectController(Controller):
|
|
452
456
|
headers[index]['X-Container-Device'] = csv_append(
|
453
457
|
headers[index].get('X-Container-Device'),
|
454
458
|
container_node['device'])
|
459
|
+
headers[index]['X-Container-Root-Db-State'] = db_state
|
455
460
|
if container_path:
|
456
461
|
headers[index]['X-Backend-Quoted-Container-Path'] = quote(
|
457
462
|
container_path)
|
@@ -626,8 +631,7 @@ class BaseObjectController(Controller):
|
|
626
631
|
int(req.headers['x-delete-at']))
|
627
632
|
x_delete_at = int(req.headers['x-delete-at'])
|
628
633
|
|
629
|
-
req.environ
|
630
|
-
'x-delete-at:%s' % x_delete_at)
|
634
|
+
append_log_info(req.environ, 'x-delete-at:%s' % x_delete_at)
|
631
635
|
|
632
636
|
delete_at_container = get_expirer_container(
|
633
637
|
x_delete_at, self.app.expiring_objects_container_divisor,
|
@@ -844,8 +848,6 @@ class BaseObjectController(Controller):
|
|
844
848
|
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
|
845
849
|
container_info['storage_policy'])
|
846
850
|
obj_ring = self.app.get_object_ring(policy_index)
|
847
|
-
container_partition, container_nodes, container_path = \
|
848
|
-
self._get_update_target(req, container_info)
|
849
851
|
partition, nodes = obj_ring.get_nodes(
|
850
852
|
self.account_name, self.container_name, self.object_name)
|
851
853
|
|
@@ -863,7 +865,7 @@ class BaseObjectController(Controller):
|
|
863
865
|
if aresp:
|
864
866
|
return aresp
|
865
867
|
|
866
|
-
if not
|
868
|
+
if not is_success(container_info.get('status')):
|
867
869
|
return HTTPNotFound(request=req)
|
868
870
|
|
869
871
|
# update content type in case it is missing
|
@@ -891,9 +893,8 @@ class BaseObjectController(Controller):
|
|
891
893
|
|
892
894
|
# add special headers to be handled by storage nodes
|
893
895
|
outgoing_headers = self._backend_requests(
|
894
|
-
req, len(nodes),
|
895
|
-
delete_at_container, delete_at_part, delete_at_nodes
|
896
|
-
container_path=container_path)
|
896
|
+
req, len(nodes), container_info,
|
897
|
+
delete_at_container, delete_at_part, delete_at_nodes)
|
897
898
|
|
898
899
|
# send object to storage nodes
|
899
900
|
resp = self._store_object(
|
@@ -916,15 +917,13 @@ class BaseObjectController(Controller):
|
|
916
917
|
next_part_power = getattr(obj_ring, 'next_part_power', None)
|
917
918
|
if next_part_power:
|
918
919
|
req.headers['X-Backend-Next-Part-Power'] = next_part_power
|
919
|
-
container_partition, container_nodes, container_path = \
|
920
|
-
self._get_update_target(req, container_info)
|
921
920
|
req.acl = container_info['write_acl']
|
922
921
|
req.environ['swift_sync_key'] = container_info['sync_key']
|
923
922
|
if 'swift.authorize' in req.environ:
|
924
923
|
aresp = req.environ['swift.authorize'](req)
|
925
924
|
if aresp:
|
926
925
|
return aresp
|
927
|
-
if not
|
926
|
+
if not is_success(container_info.get('status')):
|
928
927
|
return HTTPNotFound(request=req)
|
929
928
|
partition, nodes = obj_ring.get_nodes(
|
930
929
|
self.account_name, self.container_name, self.object_name)
|
@@ -947,9 +946,7 @@ class BaseObjectController(Controller):
|
|
947
946
|
obj_ring, partition, req, policy=policy,
|
948
947
|
local_handoffs_first=True)
|
949
948
|
|
950
|
-
headers = self._backend_requests(
|
951
|
-
req, node_count, container_partition, container_nodes,
|
952
|
-
container_path=container_path)
|
949
|
+
headers = self._backend_requests(req, node_count, container_info)
|
953
950
|
return self._delete_object(req, obj_ring, partition, headers,
|
954
951
|
node_count=node_count,
|
955
952
|
node_iterator=node_iterator)
|
swift/proxy/server.py
CHANGED
@@ -36,7 +36,8 @@ from swift.common.utils import Watchdog, get_logger, \
|
|
36
36
|
get_remote_client, split_path, config_true_value, generate_trans_id, \
|
37
37
|
affinity_key_function, affinity_locality_predicate, list_from_csv, \
|
38
38
|
parse_prefixed_conf, config_auto_int_value, node_to_string, \
|
39
|
-
config_request_node_count_value, config_percent_value, cap_length
|
39
|
+
config_request_node_count_value, config_percent_value, cap_length, \
|
40
|
+
parse_options
|
40
41
|
from swift.common.registry import register_swift_info
|
41
42
|
from swift.common.constraints import check_utf8, valid_api_version
|
42
43
|
from swift.proxy.controllers import AccountController, ContainerController, \
|
@@ -49,6 +50,7 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
|
|
49
50
|
HTTPServerError, HTTPException, Request, HTTPServiceUnavailable, \
|
50
51
|
wsgi_to_str
|
51
52
|
from swift.common.exceptions import APIVersionError
|
53
|
+
from swift.common.wsgi import run_wsgi
|
52
54
|
|
53
55
|
|
54
56
|
# List of entry points for mandatory middlewares.
|
@@ -286,6 +288,8 @@ class Application(object):
|
|
286
288
|
if a.strip()]
|
287
289
|
self.strict_cors_mode = config_true_value(
|
288
290
|
conf.get('strict_cors_mode', 't'))
|
291
|
+
self.allow_open_expired = config_true_value(
|
292
|
+
conf.get('allow_open_expired', 'f'))
|
289
293
|
self.node_timings = {}
|
290
294
|
self.timing_expiry = int(conf.get('timing_expiry', 300))
|
291
295
|
value = conf.get('request_node_count', '2 * replicas')
|
@@ -347,6 +351,7 @@ class Application(object):
|
|
347
351
|
policies=POLICIES.get_policy_info(),
|
348
352
|
allow_account_management=self.allow_account_management,
|
349
353
|
account_autocreate=self.account_autocreate,
|
354
|
+
allow_open_expired=self.allow_open_expired,
|
350
355
|
**constraints.EFFECTIVE_CONSTRAINTS)
|
351
356
|
self.watchdog = Watchdog()
|
352
357
|
self.watchdog.spawn()
|
@@ -815,3 +820,12 @@ def app_factory(global_conf, **local_conf):
|
|
815
820
|
app = Application(conf)
|
816
821
|
app.check_config()
|
817
822
|
return app
|
823
|
+
|
824
|
+
|
825
|
+
def main():
|
826
|
+
conf_file, options = parse_options(test_config=True)
|
827
|
+
sys.exit(run_wsgi(conf_file, 'proxy-server', **options))
|
828
|
+
|
829
|
+
|
830
|
+
if __name__ == '__main__':
|
831
|
+
main()
|