swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +183 -29
  17. swift/cli/manage_shard_ranges.py +708 -37
  18. swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +198 -127
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +396 -147
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -81
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +52 -19
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +192 -174
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2191 -2762
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +555 -536
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +552 -227
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +397 -176
  135. swift/container/sharder.py +1580 -639
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +213 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +653 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +452 -86
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1009 -490
  156. swift/proxy/server.py +185 -112
  157. swift-2.35.0.dist-info/AUTHORS +501 -0
  158. swift-2.35.0.dist-info/LICENSE +202 -0
  159. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
  160. swift-2.35.0.dist-info/RECORD +201 -0
  161. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  162. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  163. swift-2.35.0.dist-info/pbr.json +1 -0
  164. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  165. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  166. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  167. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  168. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  169. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  170. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  171. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  172. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  173. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  174. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  175. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  176. swift-2.23.2.data/scripts/swift-account-auditor +0 -23
  177. swift-2.23.2.data/scripts/swift-account-info +0 -51
  178. swift-2.23.2.data/scripts/swift-account-reaper +0 -23
  179. swift-2.23.2.data/scripts/swift-account-replicator +0 -34
  180. swift-2.23.2.data/scripts/swift-account-server +0 -23
  181. swift-2.23.2.data/scripts/swift-container-auditor +0 -23
  182. swift-2.23.2.data/scripts/swift-container-info +0 -51
  183. swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
  184. swift-2.23.2.data/scripts/swift-container-replicator +0 -34
  185. swift-2.23.2.data/scripts/swift-container-sharder +0 -33
  186. swift-2.23.2.data/scripts/swift-container-sync +0 -23
  187. swift-2.23.2.data/scripts/swift-container-updater +0 -23
  188. swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
  189. swift-2.23.2.data/scripts/swift-form-signature +0 -20
  190. swift-2.23.2.data/scripts/swift-init +0 -119
  191. swift-2.23.2.data/scripts/swift-object-auditor +0 -29
  192. swift-2.23.2.data/scripts/swift-object-expirer +0 -33
  193. swift-2.23.2.data/scripts/swift-object-info +0 -60
  194. swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
  195. swift-2.23.2.data/scripts/swift-object-relinker +0 -41
  196. swift-2.23.2.data/scripts/swift-object-replicator +0 -37
  197. swift-2.23.2.data/scripts/swift-object-server +0 -27
  198. swift-2.23.2.data/scripts/swift-object-updater +0 -23
  199. swift-2.23.2.data/scripts/swift-proxy-server +0 -23
  200. swift-2.23.2.data/scripts/swift-recon +0 -24
  201. swift-2.23.2.data/scripts/swift-ring-builder +0 -24
  202. swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
  203. swift-2.23.2.data/scripts/swift-ring-composer +0 -22
  204. swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
  205. swift-2.23.2.dist-info/RECORD +0 -220
  206. swift-2.23.2.dist-info/metadata.json +0 -1
  207. swift-2.23.2.dist-info/pbr.json +0 -1
  208. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -195,9 +195,8 @@ payload sent to the proxy (the list of objects/containers to be deleted).
195
195
  """
196
196
 
197
197
  import json
198
- import six
199
198
  import tarfile
200
- from xml.sax import saxutils
199
+ from xml.sax.saxutils import escape # nosec B406
201
200
  from time import time
202
201
  from eventlet import sleep
203
202
  import zlib
@@ -206,8 +205,8 @@ from swift.common.swob import Request, HTTPBadGateway, \
206
205
  HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \
207
206
  HTTPLengthRequired, HTTPException, HTTPServerError, wsgify, \
208
207
  bytes_to_wsgi, str_to_wsgi, wsgi_unquote, wsgi_quote, wsgi_to_str
209
- from swift.common.utils import get_logger, register_swift_info, \
210
- StreamingPile
208
+ from swift.common.utils import get_logger, StreamingPile
209
+ from swift.common.registry import register_swift_info
211
210
  from swift.common import constraints
212
211
  from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
213
212
  from swift.common.request_helpers import is_user_meta
@@ -247,18 +246,16 @@ def get_response_body(data_format, data_dict, error_list, root_tag):
247
246
  xml_key = key.replace(' ', '_').lower()
248
247
  output.extend([
249
248
  '<', xml_key, '>',
250
- saxutils.escape(str(data_dict[key])),
249
+ escape(str(data_dict[key])),
251
250
  '</', xml_key, '>\n',
252
251
  ])
253
252
  output.append('<errors>\n')
254
253
  for name, status in error_list:
255
254
  output.extend([
256
- '<object><name>', saxutils.escape(name), '</name><status>',
257
- saxutils.escape(status), '</status></object>\n',
255
+ '<object><name>', escape(name), '</name><status>',
256
+ escape(status), '</status></object>\n',
258
257
  ])
259
258
  output.extend(['</errors>\n</', root_tag, '>\n'])
260
- if six.PY2:
261
- return ''.join(output)
262
259
  return ''.join(output).encode('utf-8')
263
260
 
264
261
  output = []
@@ -268,8 +265,6 @@ def get_response_body(data_format, data_dict, error_list, root_tag):
268
265
  output.extend(
269
266
  '%s, %s\n' % (name, status)
270
267
  for name, status in error_list)
271
- if six.PY2:
272
- return ''.join(output)
273
268
  return ''.join(output).encode('utf-8')
274
269
 
275
270
 
@@ -279,13 +274,9 @@ def pax_key_to_swift_header(pax_key):
279
274
  return "Content-Type"
280
275
  elif pax_key.startswith(u"SCHILY.xattr.user.meta."):
281
276
  useful_part = pax_key[len(u"SCHILY.xattr.user.meta."):]
282
- if six.PY2:
283
- return "X-Object-Meta-" + useful_part.encode("utf-8")
284
277
  return str_to_wsgi("X-Object-Meta-" + useful_part)
285
278
  elif pax_key.startswith(u"LIBARCHIVE.xattr.user.meta."):
286
279
  useful_part = pax_key[len(u"LIBARCHIVE.xattr.user.meta."):]
287
- if six.PY2:
288
- return "X-Object-Meta-" + useful_part.encode("utf-8")
289
280
  return str_to_wsgi("X-Object-Meta-" + useful_part)
290
281
  else:
291
282
  # You can get things like atime/mtime/ctime or filesystem ACLs in
@@ -357,15 +348,12 @@ class Bulk(object):
357
348
  while data_remaining:
358
349
  if b'\n' in line:
359
350
  obj_to_delete, line = line.split(b'\n', 1)
360
- if six.PY2:
361
- obj_to_delete = wsgi_unquote(obj_to_delete.strip())
362
- else:
363
- # yeah, all this chaining is pretty terrible...
364
- # but it gets even worse trying to use UTF-8 and
365
- # errors='surrogateescape' when dealing with terrible
366
- # input like b'\xe2%98\x83'
367
- obj_to_delete = wsgi_to_str(wsgi_unquote(
368
- bytes_to_wsgi(obj_to_delete.strip())))
351
+ # yeah, all this chaining is pretty terrible...
352
+ # but it gets even worse trying to use UTF-8 and
353
+ # errors='surrogateescape' when dealing with terrible
354
+ # input like b'\xe2%98\x83'
355
+ obj_to_delete = wsgi_to_str(wsgi_unquote(
356
+ bytes_to_wsgi(obj_to_delete.strip())))
369
357
  objs_to_delete.append({'name': obj_to_delete})
370
358
  else:
371
359
  data = req.body_file.read(self.max_path_length)
@@ -373,11 +361,8 @@ class Bulk(object):
373
361
  line += data
374
362
  else:
375
363
  data_remaining = False
376
- if six.PY2:
377
- obj_to_delete = wsgi_unquote(line.strip())
378
- else:
379
- obj_to_delete = wsgi_to_str(wsgi_unquote(
380
- bytes_to_wsgi(line.strip())))
364
+ obj_to_delete = wsgi_to_str(wsgi_unquote(
365
+ bytes_to_wsgi(line.strip())))
381
366
  if obj_to_delete:
382
367
  objs_to_delete.append({'name': obj_to_delete})
383
368
  if len(objs_to_delete) > self.max_deletes_per_request:
@@ -457,7 +442,8 @@ class Bulk(object):
457
442
  failed_files.append([wsgi_quote(str_to_wsgi(obj_name)),
458
443
  HTTPPreconditionFailed().status])
459
444
  continue
460
- yield (obj_name, delete_path)
445
+ yield (obj_name, delete_path,
446
+ obj_to_delete.get('version_id'))
461
447
 
462
448
  def objs_then_containers(objs_to_delete):
463
449
  # process all objects first
@@ -467,13 +453,17 @@ class Bulk(object):
467
453
  yield delete_filter(lambda name: '/' not in name.strip('/'),
468
454
  objs_to_delete)
469
455
 
470
- def do_delete(obj_name, delete_path):
456
+ def do_delete(obj_name, delete_path, version_id):
471
457
  delete_obj_req = make_subrequest(
472
458
  req.environ, method='DELETE',
473
459
  path=wsgi_quote(str_to_wsgi(delete_path)),
474
460
  headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
475
461
  body='', agent='%(orig)s ' + user_agent,
476
462
  swift_source=swift_source)
463
+ if version_id is None:
464
+ delete_obj_req.params = {}
465
+ else:
466
+ delete_obj_req.params = {'version-id': version_id}
477
467
  return (delete_obj_req.get_response(self.app), obj_name, 0)
478
468
 
479
469
  with StreamingPile(self.delete_concurrency) as pile:
@@ -572,9 +562,7 @@ class Bulk(object):
572
562
  len(failed_files) >= self.max_failed_extractions:
573
563
  break
574
564
  if tar_info.isfile():
575
- obj_path = tar_info.name
576
- if not six.PY2:
577
- obj_path = obj_path.encode('utf-8', 'surrogateescape')
565
+ obj_path = tar_info.name.encode('utf-8', 'surrogateescape')
578
566
  obj_path = bytes_to_wsgi(obj_path)
579
567
  if obj_path.startswith('./'):
580
568
  obj_path = obj_path[2:]
@@ -13,8 +13,6 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
- from swift import gettext_ as _
17
-
18
16
  from swift.common.swob import Request, HTTPServerError
19
17
  from swift.common.utils import get_logger, generate_trans_id, close_if_possible
20
18
  from swift.common.wsgi import WSGIContext
@@ -74,7 +72,7 @@ class CatchErrorsContext(WSGIContext):
74
72
  # catch any errors in the pipeline
75
73
  resp = self._app_call(env)
76
74
  except: # noqa
77
- self.logger.exception(_('Error: An error occurred'))
75
+ self.logger.exception('Error: An error occurred')
78
76
  resp = HTTPServerError(request=Request(env),
79
77
  body=b'An error occurred',
80
78
  content_type='text/plain')
@@ -27,10 +27,6 @@ maximum lookup depth. If a match is found, the environment's Host header is
27
27
  rewritten and the request is passed further down the WSGI chain.
28
28
  """
29
29
 
30
- import six
31
-
32
- from swift import gettext_ as _
33
-
34
30
  try:
35
31
  import dns.resolver
36
32
  import dns.exception
@@ -44,7 +40,8 @@ from swift.common.middleware import RewriteContext
44
40
  from swift.common.swob import Request, HTTPBadRequest, \
45
41
  str_to_wsgi, wsgi_to_str
46
42
  from swift.common.utils import cache_from_env, get_logger, is_valid_ip, \
47
- list_from_csv, parse_socket_string, register_swift_info
43
+ list_from_csv, parse_socket_string
44
+ from swift.common.registry import register_swift_info
48
45
 
49
46
 
50
47
  def lookup_cname(domain, resolver): # pragma: no cover
@@ -59,7 +56,7 @@ def lookup_cname(domain, resolver): # pragma: no cover
59
56
  try:
60
57
  answer = resolver.query(domain, 'CNAME').rrset
61
58
  ttl = answer.ttl
62
- result = answer.items[0].to_text()
59
+ result = list(answer.items)[0].to_text()
63
60
  result = result.rstrip('.')
64
61
  return ttl, result
65
62
  except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
@@ -150,8 +147,6 @@ class CNAMELookupMiddleware(object):
150
147
  if self.memcache:
151
148
  memcache_key = ''.join(['cname-', a_domain])
152
149
  found_domain = self.memcache.get(memcache_key)
153
- if six.PY2 and found_domain:
154
- found_domain = found_domain.encode('utf-8')
155
150
  if found_domain is None:
156
151
  ttl, found_domain = lookup_cname(a_domain, self.resolver)
157
152
  if self.memcache and ttl > 0:
@@ -166,7 +161,7 @@ class CNAMELookupMiddleware(object):
166
161
  elif self._domain_endswith_in_storage_domain(found_domain):
167
162
  # Found it!
168
163
  self.logger.info(
169
- _('Mapped %(given_domain)s to %(found_domain)s') %
164
+ 'Mapped %(given_domain)s to %(found_domain)s',
170
165
  {'given_domain': given_domain,
171
166
  'found_domain': found_domain})
172
167
  if port:
@@ -179,8 +174,8 @@ class CNAMELookupMiddleware(object):
179
174
  else:
180
175
  # try one more deep in the chain
181
176
  self.logger.debug(
182
- _('Following CNAME chain for '
183
- '%(given_domain)s to %(found_domain)s') %
177
+ 'Following CNAME chain for '
178
+ '%(given_domain)s to %(found_domain)s',
184
179
  {'given_domain': given_domain,
185
180
  'found_domain': found_domain})
186
181
  a_domain = found_domain
@@ -54,7 +54,7 @@ For example::
54
54
  from swift.common.http import is_success
55
55
  from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \
56
56
  wsgify
57
- from swift.common.utils import register_swift_info
57
+ from swift.common.registry import register_swift_info
58
58
  from swift.proxy.controllers.base import get_container_info
59
59
 
60
60
 
@@ -15,11 +15,14 @@
15
15
 
16
16
  import os
17
17
 
18
+ from swift.common.constraints import valid_api_version
18
19
  from swift.common.container_sync_realms import ContainerSyncRealms
20
+ from swift.common.request_helpers import append_log_info
19
21
  from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify
20
22
  from swift.common.utils import (
21
- config_true_value, get_logger, register_swift_info, streq_const_time)
23
+ config_true_value, get_logger, streq_const_time)
22
24
  from swift.proxy.controllers.base import get_container_info
25
+ from swift.common.registry import register_swift_info
23
26
 
24
27
 
25
28
  class ContainerSync(object):
@@ -67,8 +70,35 @@ class ContainerSync(object):
67
70
 
68
71
  @wsgify
69
72
  def __call__(self, req):
73
+ if req.path == '/info':
74
+ # Ensure /info requests get the freshest results
75
+ self.register_info()
76
+ return self.app
77
+
78
+ try:
79
+ (version, acc, cont, obj) = req.split_path(3, 4, True)
80
+ bad_path = False
81
+ except ValueError:
82
+ bad_path = True
83
+
84
+ # use of bad_path bool is to avoid recursive tracebacks
85
+ if bad_path or not valid_api_version(version):
86
+ return self.app
87
+
88
+ # validate container-sync metdata update
89
+ info = get_container_info(
90
+ req.environ, self.app, swift_source='CS')
91
+ sync_to = req.headers.get('x-container-sync-to')
92
+ if req.method in ('PUT', 'POST') and cont and not obj:
93
+ versions_cont = info.get(
94
+ 'sysmeta', {}).get('versions-container')
95
+ if sync_to and versions_cont:
96
+ raise HTTPBadRequest(
97
+ 'Cannot configure container sync on a container '
98
+ 'with object versioning configured.',
99
+ request=req)
100
+
70
101
  if not self.allow_full_urls:
71
- sync_to = req.headers.get('x-container-sync-to')
72
102
  if sync_to and not sync_to.startswith('//'):
73
103
  raise HTTPBadRequest(
74
104
  body='Full URLs are not allowed for X-Container-Sync-To '
@@ -80,22 +110,17 @@ class ContainerSync(object):
80
110
  valid = False
81
111
  auth = auth.split()
82
112
  if len(auth) != 3:
83
- req.environ.setdefault('swift.log_info', []).append(
84
- 'cs:not-3-args')
113
+ append_log_info(req.environ, 'cs:not-3-args')
85
114
  else:
86
115
  realm, nonce, sig = auth
87
116
  realm_key = self.realms_conf.key(realm)
88
117
  realm_key2 = self.realms_conf.key2(realm)
89
118
  if not realm_key:
90
- req.environ.setdefault('swift.log_info', []).append(
91
- 'cs:no-local-realm-key')
119
+ append_log_info(req.environ, 'cs:no-local-realm-key')
92
120
  else:
93
- info = get_container_info(
94
- req.environ, self.app, swift_source='CS')
95
121
  user_key = info.get('sync_key')
96
122
  if not user_key:
97
- req.environ.setdefault('swift.log_info', []).append(
98
- 'cs:no-local-user-key')
123
+ append_log_info(req.environ, 'cs:no-local-user-key')
99
124
  else:
100
125
  # x-timestamp headers get shunted by gatekeeper
101
126
  if 'x-backend-inbound-x-timestamp' in req.headers:
@@ -112,11 +137,9 @@ class ContainerSync(object):
112
137
  realm_key2, user_key) if realm_key2 else expected
113
138
  if not streq_const_time(sig, expected) and \
114
139
  not streq_const_time(sig, expected2):
115
- req.environ.setdefault(
116
- 'swift.log_info', []).append('cs:invalid-sig')
140
+ append_log_info(req.environ, 'cs:invalid-sig')
117
141
  else:
118
- req.environ.setdefault(
119
- 'swift.log_info', []).append('cs:valid')
142
+ append_log_info(req.environ, 'cs:valid')
120
143
  valid = True
121
144
  if not valid:
122
145
  exc = HTTPUnauthorized(
@@ -134,10 +157,9 @@ class ContainerSync(object):
134
157
  # syntax and might be synced before its segments, so stop SLO
135
158
  # middleware from performing the usual manifest validation.
136
159
  req.environ['swift.slo_override'] = True
160
+ # Similar arguments for static symlinks
161
+ req.environ['swift.symlink_override'] = True
137
162
 
138
- if req.path == '/info':
139
- # Ensure /info requests get the freshest results
140
- self.register_info()
141
163
  return self.app
142
164
 
143
165
 
@@ -319,6 +319,9 @@ class ServerSideCopyMiddleware(object):
319
319
  if 'last-modified' in source_resp.headers:
320
320
  resp_headers['X-Copied-From-Last-Modified'] = \
321
321
  source_resp.headers['last-modified']
322
+ if 'X-Object-Version-Id' in source_resp.headers:
323
+ resp_headers['X-Copied-From-Version-Id'] = \
324
+ source_resp.headers['X-Object-Version-Id']
322
325
  # Existing sys and user meta of source object is added to response
323
326
  # headers in addition to the new ones.
324
327
  _copy_headers(sink_req.headers, resp_headers)
@@ -374,6 +377,8 @@ class ServerSideCopyMiddleware(object):
374
377
  sink_req.headers.update(req.headers)
375
378
 
376
379
  params = sink_req.params
380
+ params_updated = False
381
+
377
382
  if params.get('multipart-manifest') == 'get':
378
383
  if 'X-Static-Large-Object' in source_resp.headers:
379
384
  params['multipart-manifest'] = 'put'
@@ -381,6 +386,13 @@ class ServerSideCopyMiddleware(object):
381
386
  del params['multipart-manifest']
382
387
  sink_req.headers['X-Object-Manifest'] = \
383
388
  source_resp.headers['X-Object-Manifest']
389
+ params_updated = True
390
+
391
+ if 'version-id' in params:
392
+ del params['version-id']
393
+ params_updated = True
394
+
395
+ if params_updated:
384
396
  sink_req.params = params
385
397
 
386
398
  # Set swift.source, data source, content length and etag
@@ -14,7 +14,7 @@
14
14
  # limitations under the License.
15
15
 
16
16
  from swift.common.swob import Request, Response
17
- from swift.common.utils import register_swift_info
17
+ from swift.common.registry import register_swift_info
18
18
 
19
19
 
20
20
  class CrossDomainMiddleware(object):
@@ -23,20 +23,24 @@ class CrossDomainMiddleware(object):
23
23
  Cross domain middleware used to respond to requests for cross domain
24
24
  policy information.
25
25
 
26
- If the path is /crossdomain.xml it will respond with an xml cross domain
27
- policy document. This allows web pages hosted elsewhere to use client
28
- side technologies such as Flash, Java and Silverlight to interact
26
+ If the path is ``/crossdomain.xml`` it will respond with an xml cross
27
+ domain policy document. This allows web pages hosted elsewhere to use
28
+ client side technologies such as Flash, Java and Silverlight to interact
29
29
  with the Swift API.
30
30
 
31
31
  To enable this middleware, add it to the pipeline in your proxy-server.conf
32
32
  file. It should be added before any authentication (e.g., tempauth or
33
33
  keystone) middleware. In this example ellipsis (...) indicate other
34
- middleware you may have chosen to use::
34
+ middleware you may have chosen to use:
35
+
36
+ .. code:: cfg
35
37
 
36
38
  [pipeline:main]
37
39
  pipeline = ... crossdomain ... authtoken ... proxy-server
38
40
 
39
- And add a filter section, such as::
41
+ And add a filter section, such as:
42
+
43
+ .. code:: cfg
40
44
 
41
45
  [filter:crossdomain]
42
46
  use = egg:swift#crossdomain
@@ -45,13 +49,22 @@ class CrossDomainMiddleware(object):
45
49
 
46
50
  For continuation lines, put some whitespace before the continuation
47
51
  text. Ensure you put a completely blank line to terminate the
48
- cross_domain_policy value.
52
+ ``cross_domain_policy`` value.
53
+
54
+ The ``cross_domain_policy`` name/value is optional. If omitted, the policy
55
+ defaults as if you had specified:
49
56
 
50
- The cross_domain_policy name/value is optional. If omitted, the policy
51
- defaults as if you had specified::
57
+ .. code:: cfg
52
58
 
53
59
  cross_domain_policy = <allow-access-from domain="*" secure="false" />
54
60
 
61
+ .. note::
62
+
63
+ The default policy is very permissive; this is appropriate
64
+ for most public cloud deployments, but may not be appropriate
65
+ for all deployments. See also:
66
+ `CWE-942 <https://cwe.mitre.org/data/definitions/942.html>`__
67
+
55
68
 
56
69
  """
57
70
 
@@ -20,7 +20,8 @@ instance of an :class:`~swift.common.middleware.crypto.encrypter.Encrypter`.
20
20
  from swift.common.middleware.crypto.decrypter import Decrypter
21
21
  from swift.common.middleware.crypto.encrypter import Encrypter
22
22
 
23
- from swift.common.utils import config_true_value, register_swift_info
23
+ from swift.common.utils import config_true_value
24
+ from swift.common.registry import register_swift_info
24
25
 
25
26
 
26
27
  def filter_factory(global_conf, **local_conf):
@@ -14,21 +14,17 @@
14
14
  # limitations under the License.
15
15
  import base64
16
16
  import binascii
17
- import collections
18
17
  import json
19
18
  import os
20
19
 
21
20
  from cryptography.hazmat.backends import default_backend
22
21
  from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
23
- import six
24
- from six.moves.urllib import parse as urlparse
22
+ import urllib.parse
25
23
 
26
- from swift import gettext_ as _
27
24
  from swift.common.exceptions import EncryptionException, UnknownSecretIdError
28
25
  from swift.common.swob import HTTPInternalServerError
29
- from swift.common.utils import get_logger
26
+ from swift.common.utils import get_logger, parse_header
30
27
  from swift.common.wsgi import WSGIContext
31
- from cgi import parse_header
32
28
 
33
29
  CRYPTO_KEY_CALLBACK = 'swift.callback.fetch_crypto_keys'
34
30
 
@@ -161,7 +157,7 @@ class CryptoWSGIContext(WSGIContext):
161
157
  try:
162
158
  fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
163
159
  except KeyError:
164
- self.logger.exception(_('ERROR get_keys() missing callback'))
160
+ self.logger.exception('ERROR get_keys() missing callback')
165
161
  raise HTTPInternalServerError(
166
162
  "Unable to retrieve encryption keys.")
167
163
 
@@ -182,12 +178,12 @@ class CryptoWSGIContext(WSGIContext):
182
178
  self.crypto.check_key(key)
183
179
  continue
184
180
  except KeyError:
185
- self.logger.exception(_("Missing key for %r") % name)
181
+ self.logger.exception("Missing key for %r", name)
186
182
  except TypeError:
187
- self.logger.exception(_("Did not get a keys dict"))
183
+ self.logger.exception("Did not get a keys dict")
188
184
  except ValueError as e:
189
185
  # don't include the key in any messages!
190
- self.logger.exception(_("Bad key for %(name)r: %(err)s") %
186
+ self.logger.exception("Bad key for %(name)r: %(err)s",
191
187
  {'name': name, 'err': e})
192
188
  raise HTTPInternalServerError(
193
189
  "Unable to retrieve encryption keys.")
@@ -227,7 +223,7 @@ def dump_crypto_meta(crypto_meta):
227
223
  for name, value in crypto_meta.items()}
228
224
 
229
225
  # use sort_keys=True to make serialized form predictable for testing
230
- return urlparse.quote_plus(
226
+ return urllib.parse.quote_plus(
231
227
  json.dumps(b64_encode_meta(crypto_meta), sort_keys=True))
232
228
 
233
229
 
@@ -253,14 +249,14 @@ def load_crypto_meta(value, b64decode=True):
253
249
  str(name): (
254
250
  base64.b64decode(val) if name in ('iv', 'key') and b64decode
255
251
  else b64_decode_meta(val) if isinstance(val, dict)
256
- else val.encode('utf8') if six.PY2 else val)
252
+ else val)
257
253
  for name, val in crypto_meta.items()}
258
254
 
259
255
  try:
260
- if not isinstance(value, six.string_types):
256
+ if not isinstance(value, str):
261
257
  raise ValueError('crypto meta not a string')
262
- val = json.loads(urlparse.unquote_plus(value))
263
- if not isinstance(val, collections.Mapping):
258
+ val = json.loads(urllib.parse.unquote_plus(value))
259
+ if not isinstance(val, dict):
264
260
  raise ValueError('crypto meta not a Mapping')
265
261
  return b64_decode_meta(val)
266
262
  except (KeyError, ValueError, TypeError) as err:
@@ -16,7 +16,6 @@
16
16
  import base64
17
17
  import json
18
18
 
19
- from swift import gettext_ as _
20
19
  from swift.common.header_key_dict import HeaderKeyDict
21
20
  from swift.common.http import is_success
22
21
  from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
@@ -77,10 +76,10 @@ class BaseDecrypterContext(CryptoWSGIContext):
77
76
  crypto_meta['body_key'])
78
77
  except KeyError as err:
79
78
  self.logger.error(
80
- _('Error decrypting %(resp_type)s: Missing %(key)s'),
79
+ 'Error decrypting %(resp_type)s: Missing %(key)s',
81
80
  {'resp_type': self.server_type, 'key': err})
82
81
  except ValueError as err:
83
- self.logger.error(_('Error decrypting %(resp_type)s: %(reason)s'),
82
+ self.logger.error('Error decrypting %(resp_type)s: %(reason)s',
84
83
  {'resp_type': self.server_type, 'reason': err})
85
84
  raise HTTPInternalServerError(
86
85
  body='Error decrypting %s' % self.server_type,
@@ -92,7 +91,7 @@ class BaseDecrypterContext(CryptoWSGIContext):
92
91
  the value itself, otherwise return the value unmodified.
93
92
 
94
93
  A value should either be a string that does not contain the ';'
95
- character or should be of the form:
94
+ character or should be of the form::
96
95
 
97
96
  <base64-encoded ciphertext>;swift_meta=<crypto meta>
98
97
 
@@ -178,7 +177,7 @@ class DecrypterObjContext(BaseDecrypterContext):
178
177
  value, key, required, bytes_to_wsgi)
179
178
  except EncryptionException as err:
180
179
  self.logger.error(
181
- _("Error decrypting header %(header)s: %(error)s"),
180
+ "Error decrypting header %(header)s: %(error)s",
182
181
  {'header': header, 'error': err})
183
182
  raise HTTPInternalServerError(
184
183
  body='Error decrypting header',
@@ -197,7 +196,7 @@ class DecrypterObjContext(BaseDecrypterContext):
197
196
  result.append((new_prefix + short_name, decrypted_value))
198
197
  return result
199
198
 
200
- def decrypt_resp_headers(self, put_keys, post_keys):
199
+ def decrypt_resp_headers(self, put_keys, post_keys, update_cors_exposed):
201
200
  """
202
201
  Find encrypted headers and replace with the decrypted versions.
203
202
 
@@ -236,11 +235,27 @@ class DecrypterObjContext(BaseDecrypterContext):
236
235
  # that map to the same x-object-meta- header names i.e. decrypted
237
236
  # headers win over unexpected, unencrypted headers.
238
237
  if post_keys:
239
- mod_hdr_pairs.extend(self.decrypt_user_metadata(post_keys))
238
+ decrypted_meta = self.decrypt_user_metadata(post_keys)
239
+ mod_hdr_pairs.extend(decrypted_meta)
240
+ else:
241
+ decrypted_meta = []
240
242
 
241
243
  mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
242
- mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers
243
- if h.lower() not in mod_hdr_names])
244
+
245
+ found_aceh = False
246
+ for header, value in self._response_headers:
247
+ lheader = header.lower()
248
+ if lheader in mod_hdr_names:
249
+ continue
250
+ if lheader == 'access-control-expose-headers':
251
+ found_aceh = True
252
+ mod_hdr_pairs.append((header, value + ', ' + ', '.join(
253
+ meta.lower() for meta, _data in decrypted_meta)))
254
+ else:
255
+ mod_hdr_pairs.append((header, value))
256
+ if update_cors_exposed and not found_aceh:
257
+ mod_hdr_pairs.append(('Access-Control-Expose-Headers', ', '.join(
258
+ meta.lower() for meta, _data in decrypted_meta)))
244
259
  return mod_hdr_pairs
245
260
 
246
261
  def multipart_response_iter(self, resp, boundary, body_key, crypto_meta):
@@ -297,7 +312,7 @@ class DecrypterObjContext(BaseDecrypterContext):
297
312
  try:
298
313
  crypto_meta = self.get_crypto_meta(header, check)
299
314
  except EncryptionException as err:
300
- self.logger.error(_('Error decrypting object: %s'), err)
315
+ self.logger.error('Error decrypting object: %s', err)
301
316
  raise HTTPInternalServerError(
302
317
  body='Error decrypting object', content_type='text/plain')
303
318
  return crypto_meta
@@ -326,7 +341,9 @@ class DecrypterObjContext(BaseDecrypterContext):
326
341
  self._response_exc_info)
327
342
  return app_resp
328
343
 
329
- mod_resp_headers = self.decrypt_resp_headers(put_keys, post_keys)
344
+ mod_resp_headers = self.decrypt_resp_headers(
345
+ put_keys, post_keys,
346
+ update_cors_exposed=bool(req.headers.get('origin')))
330
347
 
331
348
  if put_crypto_meta and req.method == 'GET' and \
332
349
  is_success(self._get_status_int()):