swift 2.33.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. swift/account/auditor.py +11 -0
  2. swift/account/reaper.py +11 -1
  3. swift/account/replicator.py +22 -0
  4. swift/account/server.py +12 -1
  5. swift-2.33.0.data/scripts/swift-account-audit → swift/cli/account_audit.py +6 -2
  6. swift-2.33.0.data/scripts/swift-config → swift/cli/config.py +1 -1
  7. swift-2.33.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +6 -2
  8. swift-2.33.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +12 -3
  9. swift-2.33.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +6 -2
  10. swift/cli/info.py +103 -2
  11. swift-2.33.0.data/scripts/swift-oldies → swift/cli/oldies.py +6 -3
  12. swift-2.33.0.data/scripts/swift-orphans → swift/cli/orphans.py +7 -2
  13. swift/cli/recon_cron.py +5 -5
  14. swift-2.33.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  15. swift/cli/relinker.py +1 -1
  16. swift/cli/ringbuilder.py +24 -0
  17. swift/common/db.py +2 -1
  18. swift/common/db_auditor.py +2 -2
  19. swift/common/db_replicator.py +6 -0
  20. swift/common/exceptions.py +12 -0
  21. swift/common/manager.py +102 -0
  22. swift/common/memcached.py +6 -13
  23. swift/common/middleware/account_quotas.py +144 -43
  24. swift/common/middleware/backend_ratelimit.py +166 -24
  25. swift/common/middleware/catch_errors.py +1 -3
  26. swift/common/middleware/cname_lookup.py +3 -5
  27. swift/common/middleware/container_sync.py +6 -10
  28. swift/common/middleware/crypto/crypto_utils.py +4 -5
  29. swift/common/middleware/crypto/decrypter.py +4 -5
  30. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  31. swift/common/middleware/proxy_logging.py +22 -16
  32. swift/common/middleware/ratelimit.py +6 -7
  33. swift/common/middleware/recon.py +6 -7
  34. swift/common/middleware/s3api/acl_handlers.py +9 -0
  35. swift/common/middleware/s3api/controllers/multi_upload.py +1 -9
  36. swift/common/middleware/s3api/controllers/obj.py +20 -1
  37. swift/common/middleware/s3api/s3api.py +2 -0
  38. swift/common/middleware/s3api/s3request.py +171 -62
  39. swift/common/middleware/s3api/s3response.py +35 -6
  40. swift/common/middleware/s3api/s3token.py +2 -2
  41. swift/common/middleware/s3api/utils.py +1 -0
  42. swift/common/middleware/slo.py +153 -52
  43. swift/common/middleware/tempauth.py +6 -4
  44. swift/common/middleware/tempurl.py +2 -2
  45. swift/common/middleware/x_profile/exceptions.py +1 -4
  46. swift/common/middleware/x_profile/html_viewer.py +9 -10
  47. swift/common/middleware/x_profile/profile_model.py +1 -2
  48. swift/common/middleware/xprofile.py +1 -2
  49. swift/common/request_helpers.py +69 -0
  50. swift/common/statsd_client.py +207 -0
  51. swift/common/utils/__init__.py +97 -1635
  52. swift/common/utils/base.py +138 -0
  53. swift/common/utils/config.py +443 -0
  54. swift/common/utils/logs.py +999 -0
  55. swift/common/wsgi.py +11 -3
  56. swift/container/auditor.py +11 -0
  57. swift/container/backend.py +10 -10
  58. swift/container/reconciler.py +11 -2
  59. swift/container/replicator.py +22 -1
  60. swift/container/server.py +12 -1
  61. swift/container/sharder.py +36 -12
  62. swift/container/sync.py +11 -1
  63. swift/container/updater.py +11 -2
  64. swift/obj/auditor.py +18 -2
  65. swift/obj/diskfile.py +8 -6
  66. swift/obj/expirer.py +155 -36
  67. swift/obj/reconstructor.py +28 -4
  68. swift/obj/replicator.py +61 -22
  69. swift/obj/server.py +64 -36
  70. swift/obj/updater.py +11 -2
  71. swift/proxy/controllers/base.py +38 -22
  72. swift/proxy/controllers/obj.py +23 -26
  73. swift/proxy/server.py +15 -1
  74. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/AUTHORS +11 -3
  75. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/METADATA +6 -5
  76. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/RECORD +81 -107
  77. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/entry_points.txt +38 -0
  78. swift-2.34.0.dist-info/pbr.json +1 -0
  79. swift-2.33.0.data/scripts/swift-account-auditor +0 -23
  80. swift-2.33.0.data/scripts/swift-account-info +0 -52
  81. swift-2.33.0.data/scripts/swift-account-reaper +0 -23
  82. swift-2.33.0.data/scripts/swift-account-replicator +0 -34
  83. swift-2.33.0.data/scripts/swift-account-server +0 -23
  84. swift-2.33.0.data/scripts/swift-container-auditor +0 -23
  85. swift-2.33.0.data/scripts/swift-container-info +0 -59
  86. swift-2.33.0.data/scripts/swift-container-reconciler +0 -21
  87. swift-2.33.0.data/scripts/swift-container-replicator +0 -34
  88. swift-2.33.0.data/scripts/swift-container-server +0 -23
  89. swift-2.33.0.data/scripts/swift-container-sharder +0 -37
  90. swift-2.33.0.data/scripts/swift-container-sync +0 -23
  91. swift-2.33.0.data/scripts/swift-container-updater +0 -23
  92. swift-2.33.0.data/scripts/swift-dispersion-report +0 -24
  93. swift-2.33.0.data/scripts/swift-form-signature +0 -20
  94. swift-2.33.0.data/scripts/swift-init +0 -119
  95. swift-2.33.0.data/scripts/swift-object-auditor +0 -29
  96. swift-2.33.0.data/scripts/swift-object-expirer +0 -33
  97. swift-2.33.0.data/scripts/swift-object-info +0 -60
  98. swift-2.33.0.data/scripts/swift-object-reconstructor +0 -33
  99. swift-2.33.0.data/scripts/swift-object-relinker +0 -23
  100. swift-2.33.0.data/scripts/swift-object-replicator +0 -37
  101. swift-2.33.0.data/scripts/swift-object-server +0 -27
  102. swift-2.33.0.data/scripts/swift-object-updater +0 -23
  103. swift-2.33.0.data/scripts/swift-proxy-server +0 -23
  104. swift-2.33.0.data/scripts/swift-recon +0 -24
  105. swift-2.33.0.data/scripts/swift-recon-cron +0 -24
  106. swift-2.33.0.data/scripts/swift-ring-builder +0 -37
  107. swift-2.33.0.data/scripts/swift-ring-builder-analyzer +0 -22
  108. swift-2.33.0.data/scripts/swift-ring-composer +0 -22
  109. swift-2.33.0.dist-info/pbr.json +0 -1
  110. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/LICENSE +0 -0
  111. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/WHEEL +0 -0
  112. {swift-2.33.0.dist-info → swift-2.34.0.dist-info}/top_level.txt +0 -0
swift/obj/server.py CHANGED
@@ -21,6 +21,7 @@ from six.moves.urllib.parse import unquote
21
21
  import json
22
22
  import os
23
23
  import multiprocessing
24
+ import sys
24
25
  import time
25
26
  import traceback
26
27
  import socket
@@ -34,7 +35,7 @@ from swift.common.utils import public, get_logger, \
34
35
  get_expirer_container, parse_mime_headers, \
35
36
  iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
36
37
  config_auto_int_value, split_path, get_redirect_data, \
37
- normalize_timestamp, md5
38
+ normalize_timestamp, md5, parse_options
38
39
  from swift.common.bufferedhttp import http_connect
39
40
  from swift.common.constraints import check_object_creation, \
40
41
  valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
@@ -50,15 +51,18 @@ from swift.common.base_storage_server import BaseStorageServer
50
51
  from swift.common.header_key_dict import HeaderKeyDict
51
52
  from swift.common.request_helpers import get_name_and_placement, \
52
53
  is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
53
- resolve_etag_is_at_header, is_sys_meta, validate_internal_obj
54
+ resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \
55
+ is_backend_open_expired
54
56
  from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
55
57
  HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
56
58
  HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
57
59
  HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
58
60
  HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
59
61
  HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
62
+ from swift.common.wsgi import run_wsgi
60
63
  from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
61
- from swift.obj.expirer import build_task_obj
64
+ from swift.obj.expirer import build_task_obj, embed_expirer_bytes_in_ctype, \
65
+ X_DELETE_TYPE
62
66
 
63
67
 
64
68
  def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size):
@@ -268,7 +272,8 @@ class ObjectController(BaseStorageServer):
268
272
 
269
273
  def async_update(self, op, account, container, obj, host, partition,
270
274
  contdevice, headers_out, objdevice, policy,
271
- logger_thread_locals=None, container_path=None):
275
+ logger_thread_locals=None, container_path=None,
276
+ db_state=None):
272
277
  """
273
278
  Sends or saves an async update.
274
279
 
@@ -290,6 +295,8 @@ class ObjectController(BaseStorageServer):
290
295
  to which the update should be sent. If given this path will be used
291
296
  instead of constructing a path from the ``account`` and
292
297
  ``container`` params.
298
+ :param db_state: The current database state of the container as
299
+ supplied to us by the proxy.
293
300
  """
294
301
  if logger_thread_locals:
295
302
  self.logger.thread_locals = logger_thread_locals
@@ -333,7 +340,7 @@ class ObjectController(BaseStorageServer):
333
340
  '%(ip)s:%(port)s/%(dev)s (saving for async update later)',
334
341
  {'ip': ip, 'port': port, 'dev': contdevice})
335
342
  data = {'op': op, 'account': account, 'container': container,
336
- 'obj': obj, 'headers': headers_out}
343
+ 'obj': obj, 'headers': headers_out, 'db_state': db_state}
337
344
  if redirect_data:
338
345
  self.logger.debug(
339
346
  'Update to %(path)s redirected to %(redirect)s',
@@ -367,6 +374,7 @@ class ObjectController(BaseStorageServer):
367
374
  contdevices = [d.strip() for d in
368
375
  headers_in.get('X-Container-Device', '').split(',')]
369
376
  contpartition = headers_in.get('X-Container-Partition', '')
377
+ contdbstate = headers_in.get('X-Container-Root-Db-State')
370
378
 
371
379
  if len(conthosts) != len(contdevices):
372
380
  # This shouldn't happen unless there's a bug in the proxy,
@@ -417,7 +425,7 @@ class ObjectController(BaseStorageServer):
417
425
  conthost, contpartition, contdevice, headers_out,
418
426
  objdevice, policy,
419
427
  logger_thread_locals=self.logger.thread_locals,
420
- container_path=contpath)
428
+ container_path=contpath, db_state=contdbstate)
421
429
  update_greenthreads.append(gt)
422
430
  # Wait a little bit to see if the container updates are successful.
423
431
  # If we immediately return after firing off the greenthread above, then
@@ -436,7 +444,7 @@ class ObjectController(BaseStorageServer):
436
444
  self.container_update_timeout, updates)
437
445
 
438
446
  def delete_at_update(self, op, delete_at, account, container, obj,
439
- request, objdevice, policy):
447
+ request, objdevice, policy, extra_headers=None):
440
448
  """
441
449
  Update the expiring objects container when objects are updated.
442
450
 
@@ -448,6 +456,7 @@ class ObjectController(BaseStorageServer):
448
456
  :param request: the original request driving the update
449
457
  :param objdevice: device name that the object is in
450
458
  :param policy: the BaseStoragePolicy instance (used for tmp dir)
459
+ :param extra_headers: dict of additional headers for the update
451
460
  """
452
461
  if config_true_value(
453
462
  request.headers.get('x-backend-replication', 'f')):
@@ -493,8 +502,10 @@ class ObjectController(BaseStorageServer):
493
502
  if not updates:
494
503
  updates = [(None, None)]
495
504
  headers_out['x-size'] = '0'
496
- headers_out['x-content-type'] = 'text/plain'
505
+ headers_out['x-content-type'] = X_DELETE_TYPE
497
506
  headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
507
+ if extra_headers:
508
+ headers_out.update(extra_headers)
498
509
  else:
499
510
  if not config_true_value(
500
511
  request.headers.get(
@@ -619,6 +630,24 @@ class ObjectController(BaseStorageServer):
619
630
  override = key.lower().replace(override_prefix, 'x-')
620
631
  update_headers[override] = val
621
632
 
633
+ def _conditional_delete_at_update(self, request, device, account,
634
+ container, obj, policy, metadata,
635
+ orig_delete_at, new_delete_at):
636
+ if new_delete_at:
637
+ extra_headers = {
638
+ 'x-content-type': embed_expirer_bytes_in_ctype(
639
+ X_DELETE_TYPE, metadata),
640
+ 'x-content-type-timestamp':
641
+ metadata.get('X-Timestamp'),
642
+ }
643
+ self.delete_at_update(
644
+ 'PUT', new_delete_at, account, container, obj, request,
645
+ device, policy, extra_headers)
646
+ if orig_delete_at and orig_delete_at != new_delete_at:
647
+ self.delete_at_update(
648
+ 'DELETE', orig_delete_at, account, container, obj,
649
+ request, device, policy)
650
+
622
651
  @public
623
652
  @timing_stats()
624
653
  def POST(self, request):
@@ -635,8 +664,7 @@ class ObjectController(BaseStorageServer):
635
664
  try:
636
665
  disk_file = self.get_diskfile(
637
666
  device, partition, account, container, obj,
638
- policy=policy, open_expired=config_true_value(
639
- request.headers.get('x-backend-replication', 'false')),
667
+ policy=policy, open_expired=is_backend_open_expired(request),
640
668
  next_part_power=next_part_power)
641
669
  except DiskFileDeviceUnavailable:
642
670
  return HTTPInsufficientStorage(drive=device, request=request)
@@ -675,15 +703,11 @@ class ObjectController(BaseStorageServer):
675
703
  wsgi_to_bytes(header_key).title())
676
704
  metadata[header_caps] = request.headers[header_key]
677
705
  orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
678
- if orig_delete_at != new_delete_at:
679
- if new_delete_at:
680
- self.delete_at_update(
681
- 'PUT', new_delete_at, account, container, obj, request,
682
- device, policy)
683
- if orig_delete_at:
684
- self.delete_at_update('DELETE', orig_delete_at, account,
685
- container, obj, request, device,
686
- policy)
706
+ disk_file_metadata = disk_file.get_datafile_metadata()
707
+ self._conditional_delete_at_update(
708
+ request, device, account, container, obj, policy,
709
+ disk_file_metadata, orig_delete_at, new_delete_at
710
+ )
687
711
  else:
688
712
  # preserve existing metadata, only content-type may be updated
689
713
  metadata = dict(disk_file.get_metafile_metadata())
@@ -993,15 +1017,10 @@ class ObjectController(BaseStorageServer):
993
1017
  orig_metadata, footers_metadata, metadata):
994
1018
  orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
995
1019
  new_delete_at = int(request.headers.get('X-Delete-At') or 0)
996
- if orig_delete_at != new_delete_at:
997
- if new_delete_at:
998
- self.delete_at_update(
999
- 'PUT', new_delete_at, account, container, obj, request,
1000
- device, policy)
1001
- if orig_delete_at:
1002
- self.delete_at_update(
1003
- 'DELETE', orig_delete_at, account, container, obj,
1004
- request, device, policy)
1020
+
1021
+ self._conditional_delete_at_update(request, device, account, container,
1022
+ obj, policy, metadata,
1023
+ orig_delete_at, new_delete_at)
1005
1024
 
1006
1025
  update_headers = HeaderKeyDict({
1007
1026
  'x-size': metadata['Content-Length'],
@@ -1074,8 +1093,7 @@ class ObjectController(BaseStorageServer):
1074
1093
  disk_file = self.get_diskfile(
1075
1094
  device, partition, account, container, obj,
1076
1095
  policy=policy, frag_prefs=frag_prefs,
1077
- open_expired=config_true_value(
1078
- request.headers.get('x-backend-replication', 'false')))
1096
+ open_expired=is_backend_open_expired(request))
1079
1097
  except DiskFileDeviceUnavailable:
1080
1098
  return HTTPInsufficientStorage(drive=device, request=request)
1081
1099
  try:
@@ -1157,8 +1175,7 @@ class ObjectController(BaseStorageServer):
1157
1175
  disk_file = self.get_diskfile(
1158
1176
  device, partition, account, container, obj,
1159
1177
  policy=policy, frag_prefs=frag_prefs,
1160
- open_expired=config_true_value(
1161
- request.headers.get('x-backend-replication', 'false')))
1178
+ open_expired=is_backend_open_expired(request))
1162
1179
  except DiskFileDeviceUnavailable:
1163
1180
  return HTTPInsufficientStorage(drive=device, request=request)
1164
1181
  try:
@@ -1264,10 +1281,10 @@ class ObjectController(BaseStorageServer):
1264
1281
  else:
1265
1282
  # differentiate success from no object at all
1266
1283
  response_class = HTTPNoContent
1267
- if orig_delete_at:
1268
- self.delete_at_update('DELETE', orig_delete_at, account,
1269
- container, obj, request, device,
1270
- policy)
1284
+ self._conditional_delete_at_update(
1285
+ request, device, account, container, obj, policy, {},
1286
+ orig_delete_at, 0
1287
+ )
1271
1288
  if orig_timestamp < req_timestamp:
1272
1289
  try:
1273
1290
  disk_file.delete(req_timestamp)
@@ -1443,3 +1460,14 @@ def app_factory(global_conf, **local_conf):
1443
1460
  conf = global_conf.copy()
1444
1461
  conf.update(local_conf)
1445
1462
  return ObjectController(conf)
1463
+
1464
+
1465
+ def main():
1466
+ conf_file, options = parse_options(test_config=True)
1467
+ sys.exit(run_wsgi(conf_file, 'object-server',
1468
+ global_conf_callback=global_conf_callback,
1469
+ **options))
1470
+
1471
+
1472
+ if __name__ == '__main__':
1473
+ main()
swift/obj/updater.py CHANGED
@@ -34,8 +34,8 @@ from swift.common.utils import get_logger, renamer, write_pickle, \
34
34
  dump_recon_cache, config_true_value, RateLimitedIterator, split_path, \
35
35
  eventlet_monkey_patch, get_redirect_data, ContextPool, hash_path, \
36
36
  non_negative_float, config_positive_int_value, non_negative_int, \
37
- EventletRateLimiter, node_to_string
38
- from swift.common.daemon import Daemon
37
+ EventletRateLimiter, node_to_string, parse_options
38
+ from swift.common.daemon import Daemon, run_daemon
39
39
  from swift.common.header_key_dict import HeaderKeyDict
40
40
  from swift.common.storage_policy import split_policy_string, PolicyError
41
41
  from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -758,3 +758,12 @@ class ObjectUpdater(Daemon):
758
758
  self.logger.timing('updater.timing.status.%s' % status,
759
759
  elapsed * 1000)
760
760
  return HTTP_INTERNAL_SERVER_ERROR, node['id'], redirect
761
+
762
+
763
+ def main():
764
+ conf_file, options = parse_options(once=True)
765
+ run_daemon(ObjectUpdater, conf_file, **options)
766
+
767
+
768
+ if __name__ == '__main__':
769
+ main()
@@ -45,7 +45,7 @@ from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
45
45
  public, split_path, list_from_csv, GreenthreadSafeIterator, \
46
46
  GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
47
47
  document_iters_to_http_response_body, cache_from_env, \
48
- CooperativeIterator, NamespaceBoundList, Namespace
48
+ CooperativeIterator, NamespaceBoundList, Namespace, ClosingMapper
49
49
  from swift.common.bufferedhttp import http_connect
50
50
  from swift.common import constraints
51
51
  from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
@@ -1107,6 +1107,18 @@ def is_good_source(status, server_type):
1107
1107
  return is_success(status) or is_redirection(status)
1108
1108
 
1109
1109
 
1110
+ def is_useful_response(resp, node):
1111
+ if not resp:
1112
+ return False
1113
+ if ('handoff_index' in node
1114
+ and resp.status == 404
1115
+ and resp.headers.get('x-backend-timestamp') is None):
1116
+ # a 404 from a handoff are not considered authoritative unless they
1117
+ # have an x-backend-timestamp that indicates that there is a tombstone
1118
+ return False
1119
+ return True
1120
+
1121
+
1110
1122
  class ByteCountEnforcer(object):
1111
1123
  """
1112
1124
  Enforces that successive calls to file_like.read() give at least
@@ -1713,7 +1725,7 @@ class GetOrHeadHandler(GetterBase):
1713
1725
  return response_part
1714
1726
 
1715
1727
  return document_iters_to_http_response_body(
1716
- (add_content_type(pi) for pi in parts_iter),
1728
+ ClosingMapper(add_content_type, parts_iter),
1717
1729
  boundary, is_multipart, self.logger)
1718
1730
 
1719
1731
  def get_working_response(self):
@@ -1956,7 +1968,7 @@ class Controller(object):
1956
1968
  def generate_request_headers(self, orig_req=None, additional=None,
1957
1969
  transfer=False):
1958
1970
  """
1959
- Create a list of headers to be used in backend requests
1971
+ Create a dict of headers to be used in backend requests
1960
1972
 
1961
1973
  :param orig_req: the original request sent by the client to the proxy
1962
1974
  :param additional: additional headers to send to the backend
@@ -2088,14 +2100,14 @@ class Controller(object):
2088
2100
  if (self.app.check_response(node, self.server_type, resp,
2089
2101
  method, path)
2090
2102
  and not is_informational(resp.status)):
2091
- return resp.status, resp.reason, resp.getheaders(), \
2092
- resp.read()
2103
+ return resp, resp.read(), node
2093
2104
 
2094
2105
  except (Exception, Timeout):
2095
2106
  self.app.exception_occurred(
2096
2107
  node, self.server_type,
2097
2108
  'Trying to %(method)s %(path)s' %
2098
2109
  {'method': method, 'path': path})
2110
+ return None, None, None
2099
2111
 
2100
2112
  def make_requests(self, req, ring, part, method, path, headers,
2101
2113
  query_string='', overrides=None, node_count=None,
@@ -2118,6 +2130,8 @@ class Controller(object):
2118
2130
  the returned status of a request.
2119
2131
  :param node_count: optional number of nodes to send request to.
2120
2132
  :param node_iterator: optional node iterator.
2133
+ :param body: byte string to use as the request body.
2134
+ Try to keep it small.
2121
2135
  :returns: a swob.Response object
2122
2136
  """
2123
2137
  nodes = GreenthreadSafeIterator(node_iterator or NodeIter(
@@ -2128,25 +2142,25 @@ class Controller(object):
2128
2142
  for head in headers:
2129
2143
  pile.spawn(self._make_request, nodes, part, method, path,
2130
2144
  head, query_string, body, self.logger.thread_locals)
2131
- response = []
2145
+ results = []
2132
2146
  statuses = []
2133
- for resp in pile:
2134
- if not resp:
2147
+ for resp, body, node in pile:
2148
+ if not is_useful_response(resp, node):
2135
2149
  continue
2136
- response.append(resp)
2137
- statuses.append(resp[0])
2150
+ results.append((resp.status, resp.reason, resp.getheaders(), body))
2151
+ statuses.append(resp.status)
2138
2152
  if self.have_quorum(statuses, node_number):
2139
2153
  break
2140
2154
  # give any pending requests *some* chance to finish
2141
2155
  finished_quickly = pile.waitall(self.app.post_quorum_timeout)
2142
- for resp in finished_quickly:
2143
- if not resp:
2156
+ for resp, body, node in finished_quickly:
2157
+ if not is_useful_response(resp, node):
2144
2158
  continue
2145
- response.append(resp)
2146
- statuses.append(resp[0])
2147
- while len(response) < node_number:
2148
- response.append((HTTP_SERVICE_UNAVAILABLE, '', '', b''))
2149
- statuses, reasons, resp_headers, bodies = zip(*response)
2159
+ results.append((resp.status, resp.reason, resp.getheaders(), body))
2160
+ statuses.append(resp.status)
2161
+ while len(results) < node_number:
2162
+ results.append((HTTP_SERVICE_UNAVAILABLE, '', '', b''))
2163
+ statuses, reasons, resp_headers, bodies = zip(*results)
2150
2164
  return self.best_response(req, statuses, reasons, bodies,
2151
2165
  '%s %s' % (self.server_type, req.method),
2152
2166
  overrides=overrides, headers=resp_headers)
@@ -2452,9 +2466,10 @@ class Controller(object):
2452
2466
 
2453
2467
  def _parse_listing_response(self, req, response):
2454
2468
  if not is_success(response.status_int):
2469
+ record_type = req.headers.get('X-Backend-Record-Type')
2455
2470
  self.logger.warning(
2456
- 'Failed to get container listing from %s: %s',
2457
- req.path_qs, response.status_int)
2471
+ 'Failed to get container %s listing from %s: %s',
2472
+ record_type, req.path_qs, response.status_int)
2458
2473
  return None
2459
2474
 
2460
2475
  try:
@@ -2463,9 +2478,10 @@ class Controller(object):
2463
2478
  raise ValueError('not a list')
2464
2479
  return data
2465
2480
  except ValueError as err:
2481
+ record_type = response.headers.get('X-Backend-Record-Type')
2466
2482
  self.logger.error(
2467
- 'Problem with listing response from %s: %r',
2468
- req.path_qs, err)
2483
+ 'Problem with container %s listing response from %s: %r',
2484
+ record_type, req.path_qs, err)
2469
2485
  return None
2470
2486
 
2471
2487
  def _get_container_listing(self, req, account, container, headers=None,
@@ -2495,7 +2511,7 @@ class Controller(object):
2495
2511
  self.logger.debug(
2496
2512
  'Get listing from %s %s' % (subreq.path_qs, headers))
2497
2513
  response = self.app.handle_request(subreq)
2498
- data = self._parse_listing_response(req, response)
2514
+ data = self._parse_listing_response(subreq, response)
2499
2515
  return data, response
2500
2516
 
2501
2517
  def _parse_namespaces(self, req, listing, response):
@@ -77,7 +77,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
77
77
  HTTPRequestedRangeNotSatisfiable, Range, HTTPInternalServerError, \
78
78
  normalize_etag, str_to_wsgi
79
79
  from swift.common.request_helpers import update_etag_is_at_header, \
80
- resolve_etag_is_at_header, validate_internal_obj, get_ip_port
80
+ resolve_etag_is_at_header, validate_internal_obj, get_ip_port, \
81
+ is_open_expired, append_log_info
81
82
 
82
83
 
83
84
  def check_content_type(req):
@@ -250,6 +251,8 @@ class BaseObjectController(Controller):
250
251
  policy = POLICIES.get_by_index(policy_index)
251
252
  obj_ring = self.app.get_object_ring(policy_index)
252
253
  req.headers['X-Backend-Storage-Policy-Index'] = policy_index
254
+ if is_open_expired(self.app, req):
255
+ req.headers['X-Backend-Open-Expired'] = 'true'
253
256
  if 'swift.authorize' in req.environ:
254
257
  aresp = req.environ['swift.authorize'](req)
255
258
  if aresp:
@@ -388,9 +391,10 @@ class BaseObjectController(Controller):
388
391
  if update_shard_ns:
389
392
  partition, nodes = self.app.container_ring.get_nodes(
390
393
  update_shard_ns.account, update_shard_ns.container)
391
- return partition, nodes, update_shard_ns.name
394
+ return partition, nodes, update_shard_ns.name, db_state
392
395
 
393
- return container_info['partition'], container_info['nodes'], None
396
+ return (container_info['partition'], container_info['nodes'], None,
397
+ db_state)
394
398
 
395
399
  @public
396
400
  @cors_validation
@@ -399,14 +403,14 @@ class BaseObjectController(Controller):
399
403
  """HTTP POST request handler."""
400
404
  container_info = self.container_info(
401
405
  self.account_name, self.container_name, req)
402
- container_partition, container_nodes, container_path = \
403
- self._get_update_target(req, container_info)
404
406
  req.acl = container_info['write_acl']
407
+ if is_open_expired(self.app, req):
408
+ req.headers['X-Backend-Open-Expired'] = 'true'
405
409
  if 'swift.authorize' in req.environ:
406
410
  aresp = req.environ['swift.authorize'](req)
407
411
  if aresp:
408
412
  return aresp
409
- if not container_nodes:
413
+ if not is_success(container_info.get('status')):
410
414
  return HTTPNotFound(request=req)
411
415
  error_response = check_metadata(req, 'object')
412
416
  if error_response:
@@ -429,17 +433,17 @@ class BaseObjectController(Controller):
429
433
  self.account_name, self.container_name, self.object_name)
430
434
 
431
435
  headers = self._backend_requests(
432
- req, len(nodes), container_partition, container_nodes,
433
- delete_at_container, delete_at_part, delete_at_nodes,
434
- container_path=container_path)
436
+ req, len(nodes), container_info, delete_at_container,
437
+ delete_at_part, delete_at_nodes)
435
438
  return self._post_object(req, obj_ring, partition, headers)
436
439
 
437
440
  def _backend_requests(self, req, n_outgoing,
438
- container_partition, containers,
439
- delete_at_container=None, delete_at_partition=None,
440
- delete_at_nodes=None, container_path=None):
441
+ container_info, delete_at_container=None,
442
+ delete_at_partition=None, delete_at_nodes=None):
441
443
  policy_index = req.headers['X-Backend-Storage-Policy-Index']
442
444
  policy = POLICIES.get_by_index(policy_index)
445
+ container_partition, containers, container_path, db_state = \
446
+ self._get_update_target(req, container_info)
443
447
  headers = [self.generate_request_headers(req, additional=req.headers)
444
448
  for _junk in range(n_outgoing)]
445
449
 
@@ -452,6 +456,7 @@ class BaseObjectController(Controller):
452
456
  headers[index]['X-Container-Device'] = csv_append(
453
457
  headers[index].get('X-Container-Device'),
454
458
  container_node['device'])
459
+ headers[index]['X-Container-Root-Db-State'] = db_state
455
460
  if container_path:
456
461
  headers[index]['X-Backend-Quoted-Container-Path'] = quote(
457
462
  container_path)
@@ -626,8 +631,7 @@ class BaseObjectController(Controller):
626
631
  int(req.headers['x-delete-at']))
627
632
  x_delete_at = int(req.headers['x-delete-at'])
628
633
 
629
- req.environ.setdefault('swift.log_info', []).append(
630
- 'x-delete-at:%s' % x_delete_at)
634
+ append_log_info(req.environ, 'x-delete-at:%s' % x_delete_at)
631
635
 
632
636
  delete_at_container = get_expirer_container(
633
637
  x_delete_at, self.app.expiring_objects_container_divisor,
@@ -844,8 +848,6 @@ class BaseObjectController(Controller):
844
848
  policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
845
849
  container_info['storage_policy'])
846
850
  obj_ring = self.app.get_object_ring(policy_index)
847
- container_partition, container_nodes, container_path = \
848
- self._get_update_target(req, container_info)
849
851
  partition, nodes = obj_ring.get_nodes(
850
852
  self.account_name, self.container_name, self.object_name)
851
853
 
@@ -863,7 +865,7 @@ class BaseObjectController(Controller):
863
865
  if aresp:
864
866
  return aresp
865
867
 
866
- if not container_nodes:
868
+ if not is_success(container_info.get('status')):
867
869
  return HTTPNotFound(request=req)
868
870
 
869
871
  # update content type in case it is missing
@@ -891,9 +893,8 @@ class BaseObjectController(Controller):
891
893
 
892
894
  # add special headers to be handled by storage nodes
893
895
  outgoing_headers = self._backend_requests(
894
- req, len(nodes), container_partition, container_nodes,
895
- delete_at_container, delete_at_part, delete_at_nodes,
896
- container_path=container_path)
896
+ req, len(nodes), container_info,
897
+ delete_at_container, delete_at_part, delete_at_nodes)
897
898
 
898
899
  # send object to storage nodes
899
900
  resp = self._store_object(
@@ -916,15 +917,13 @@ class BaseObjectController(Controller):
916
917
  next_part_power = getattr(obj_ring, 'next_part_power', None)
917
918
  if next_part_power:
918
919
  req.headers['X-Backend-Next-Part-Power'] = next_part_power
919
- container_partition, container_nodes, container_path = \
920
- self._get_update_target(req, container_info)
921
920
  req.acl = container_info['write_acl']
922
921
  req.environ['swift_sync_key'] = container_info['sync_key']
923
922
  if 'swift.authorize' in req.environ:
924
923
  aresp = req.environ['swift.authorize'](req)
925
924
  if aresp:
926
925
  return aresp
927
- if not container_nodes:
926
+ if not is_success(container_info.get('status')):
928
927
  return HTTPNotFound(request=req)
929
928
  partition, nodes = obj_ring.get_nodes(
930
929
  self.account_name, self.container_name, self.object_name)
@@ -947,9 +946,7 @@ class BaseObjectController(Controller):
947
946
  obj_ring, partition, req, policy=policy,
948
947
  local_handoffs_first=True)
949
948
 
950
- headers = self._backend_requests(
951
- req, node_count, container_partition, container_nodes,
952
- container_path=container_path)
949
+ headers = self._backend_requests(req, node_count, container_info)
953
950
  return self._delete_object(req, obj_ring, partition, headers,
954
951
  node_count=node_count,
955
952
  node_iterator=node_iterator)
swift/proxy/server.py CHANGED
@@ -36,7 +36,8 @@ from swift.common.utils import Watchdog, get_logger, \
36
36
  get_remote_client, split_path, config_true_value, generate_trans_id, \
37
37
  affinity_key_function, affinity_locality_predicate, list_from_csv, \
38
38
  parse_prefixed_conf, config_auto_int_value, node_to_string, \
39
- config_request_node_count_value, config_percent_value, cap_length
39
+ config_request_node_count_value, config_percent_value, cap_length, \
40
+ parse_options
40
41
  from swift.common.registry import register_swift_info
41
42
  from swift.common.constraints import check_utf8, valid_api_version
42
43
  from swift.proxy.controllers import AccountController, ContainerController, \
@@ -49,6 +50,7 @@ from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
49
50
  HTTPServerError, HTTPException, Request, HTTPServiceUnavailable, \
50
51
  wsgi_to_str
51
52
  from swift.common.exceptions import APIVersionError
53
+ from swift.common.wsgi import run_wsgi
52
54
 
53
55
 
54
56
  # List of entry points for mandatory middlewares.
@@ -286,6 +288,8 @@ class Application(object):
286
288
  if a.strip()]
287
289
  self.strict_cors_mode = config_true_value(
288
290
  conf.get('strict_cors_mode', 't'))
291
+ self.allow_open_expired = config_true_value(
292
+ conf.get('allow_open_expired', 'f'))
289
293
  self.node_timings = {}
290
294
  self.timing_expiry = int(conf.get('timing_expiry', 300))
291
295
  value = conf.get('request_node_count', '2 * replicas')
@@ -347,6 +351,7 @@ class Application(object):
347
351
  policies=POLICIES.get_policy_info(),
348
352
  allow_account_management=self.allow_account_management,
349
353
  account_autocreate=self.account_autocreate,
354
+ allow_open_expired=self.allow_open_expired,
350
355
  **constraints.EFFECTIVE_CONSTRAINTS)
351
356
  self.watchdog = Watchdog()
352
357
  self.watchdog.spawn()
@@ -815,3 +820,12 @@ def app_factory(global_conf, **local_conf):
815
820
  app = Application(conf)
816
821
  app.check_config()
817
822
  return app
823
+
824
+
825
+ def main():
826
+ conf_file, options = parse_options(test_config=True)
827
+ sys.exit(run_wsgi(conf_file, 'proxy-server', **options))
828
+
829
+
830
+ if __name__ == '__main__':
831
+ main()