swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.3.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.3.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.3.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +154 -14
  17. swift/cli/manage_shard_ranges.py +705 -37
  18. swift-2.23.3.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.3.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.3.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.3.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +195 -128
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +390 -145
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -95
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +51 -18
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +191 -173
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.3.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2163 -2772
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +553 -535
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +490 -231
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +358 -165
  135. swift/container/sharder.py +1540 -684
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +207 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +652 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +433 -92
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1000 -489
  156. swift/proxy/server.py +185 -112
  157. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/AUTHORS +58 -11
  158. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/METADATA +51 -56
  159. swift-2.35.0.dist-info/RECORD +201 -0
  160. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  161. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  162. swift-2.35.0.dist-info/pbr.json +1 -0
  163. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  164. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  165. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  166. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  167. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  168. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  169. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  170. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  171. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  172. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  173. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  174. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  175. swift-2.23.3.data/scripts/swift-account-auditor +0 -23
  176. swift-2.23.3.data/scripts/swift-account-info +0 -51
  177. swift-2.23.3.data/scripts/swift-account-reaper +0 -23
  178. swift-2.23.3.data/scripts/swift-account-replicator +0 -34
  179. swift-2.23.3.data/scripts/swift-account-server +0 -23
  180. swift-2.23.3.data/scripts/swift-container-auditor +0 -23
  181. swift-2.23.3.data/scripts/swift-container-info +0 -55
  182. swift-2.23.3.data/scripts/swift-container-reconciler +0 -21
  183. swift-2.23.3.data/scripts/swift-container-replicator +0 -34
  184. swift-2.23.3.data/scripts/swift-container-sharder +0 -37
  185. swift-2.23.3.data/scripts/swift-container-sync +0 -23
  186. swift-2.23.3.data/scripts/swift-container-updater +0 -23
  187. swift-2.23.3.data/scripts/swift-dispersion-report +0 -24
  188. swift-2.23.3.data/scripts/swift-form-signature +0 -20
  189. swift-2.23.3.data/scripts/swift-init +0 -119
  190. swift-2.23.3.data/scripts/swift-object-auditor +0 -29
  191. swift-2.23.3.data/scripts/swift-object-expirer +0 -33
  192. swift-2.23.3.data/scripts/swift-object-info +0 -60
  193. swift-2.23.3.data/scripts/swift-object-reconstructor +0 -33
  194. swift-2.23.3.data/scripts/swift-object-relinker +0 -41
  195. swift-2.23.3.data/scripts/swift-object-replicator +0 -37
  196. swift-2.23.3.data/scripts/swift-object-server +0 -27
  197. swift-2.23.3.data/scripts/swift-object-updater +0 -23
  198. swift-2.23.3.data/scripts/swift-proxy-server +0 -23
  199. swift-2.23.3.data/scripts/swift-recon +0 -24
  200. swift-2.23.3.data/scripts/swift-ring-builder +0 -24
  201. swift-2.23.3.data/scripts/swift-ring-builder-analyzer +0 -22
  202. swift-2.23.3.data/scripts/swift-ring-composer +0 -22
  203. swift-2.23.3.dist-info/RECORD +0 -220
  204. swift-2.23.3.dist-info/pbr.json +0 -1
  205. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/LICENSE +0 -0
  206. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -25,9 +25,9 @@ from swift.common.request_helpers import get_object_transient_sysmeta, \
25
25
  strip_user_meta_prefix, is_user_meta, update_etag_is_at_header, \
26
26
  get_container_update_override_key
27
27
  from swift.common.swob import Request, Match, HTTPException, \
28
- HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi
28
+ HTTPUnprocessableEntity, wsgi_to_bytes, bytes_to_wsgi, normalize_etag
29
29
  from swift.common.utils import get_logger, config_true_value, \
30
- MD5_OF_EMPTY_STRING
30
+ MD5_OF_EMPTY_STRING, md5, InputProxy
31
31
 
32
32
 
33
33
  def encrypt_header_val(crypto, value, key):
@@ -66,11 +66,11 @@ def _hmac_etag(key, etag):
66
66
  return base64.b64encode(result).decode()
67
67
 
68
68
 
69
- class EncInputWrapper(object):
69
+ class EncInputWrapper(InputProxy):
70
70
  """File-like object to be swapped in for wsgi.input."""
71
71
  def __init__(self, crypto, keys, req, logger):
72
+ super().__init__(req.environ['wsgi.input'])
72
73
  self.env = req.environ
73
- self.wsgi_input = req.environ['wsgi.input']
74
74
  self.path = req.path
75
75
  self.crypto = crypto
76
76
  self.body_crypto_ctxt = None
@@ -91,8 +91,8 @@ class EncInputWrapper(object):
91
91
  self.body_crypto_meta['key_id'] = self.keys['id']
92
92
  self.body_crypto_ctxt = self.crypto.create_encryption_ctxt(
93
93
  body_key, self.body_crypto_meta.get('iv'))
94
- self.plaintext_md5 = hashlib.md5()
95
- self.ciphertext_md5 = hashlib.md5()
94
+ self.plaintext_md5 = md5(usedforsecurity=False)
95
+ self.ciphertext_md5 = md5(usedforsecurity=False)
96
96
 
97
97
  def install_footers_callback(self, req):
98
98
  # the proxy controller will call back for footer metadata after
@@ -166,7 +166,7 @@ class EncInputWrapper(object):
166
166
  # value, with the crypto parameters appended. We use the
167
167
  # container key here so that only that key is required to
168
168
  # decrypt all etag values in a container listing when handling
169
- # a container GET request. Don't encrypt an EMPTY_ETAG
169
+ # a container GET request. Don't encrypt an MD5_OF_EMPTY_STRING
170
170
  # unless there actually was some body content, in which case
171
171
  # the container-listing etag is possibly conveying some
172
172
  # non-obvious information.
@@ -180,15 +180,7 @@ class EncInputWrapper(object):
180
180
 
181
181
  req.environ['swift.callback.update_footers'] = footers_callback
182
182
 
183
- def read(self, *args, **kwargs):
184
- return self.readChunk(self.wsgi_input.read, *args, **kwargs)
185
-
186
- def readline(self, *args, **kwargs):
187
- return self.readChunk(self.wsgi_input.readline, *args, **kwargs)
188
-
189
- def readChunk(self, read_method, *args, **kwargs):
190
- chunk = read_method(*args, **kwargs)
191
-
183
+ def chunk_update(self, chunk, eof, *args, **kwargs):
192
184
  if chunk:
193
185
  self._init_encryption_context()
194
186
  self.plaintext_md5.update(chunk)
@@ -263,7 +255,7 @@ class EncrypterObjContext(CryptoWSGIContext):
263
255
  ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest()
264
256
  mod_resp_headers = [
265
257
  (h, v if (h.lower() != 'etag' or
266
- v.strip('"') != ciphertext_etag)
258
+ normalize_etag(v) != ciphertext_etag)
267
259
  else plaintext_etag)
268
260
  for h, v in mod_resp_headers]
269
261
 
@@ -369,7 +361,10 @@ class Encrypter(object):
369
361
  return self.app(env, start_response)
370
362
  try:
371
363
  req.split_path(4, 4, True)
364
+ is_object_request = True
372
365
  except ValueError:
366
+ is_object_request = False
367
+ if not is_object_request:
373
368
  return self.app(env, start_response)
374
369
 
375
370
  if req.method in ('GET', 'HEAD'):
@@ -14,7 +14,6 @@
14
14
  # limitations under the License.
15
15
  import hashlib
16
16
  import hmac
17
- import six
18
17
 
19
18
  from swift.common.exceptions import UnknownSecretIdError
20
19
  from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
@@ -101,14 +100,9 @@ class KeyMasterContext(WSGIContext):
101
100
  # Older py3 proxies may have written down crypto meta as WSGI
102
101
  # strings; we still need to be able to read that
103
102
  try:
104
- if six.PY2:
105
- alt_path = tuple(
106
- part.decode('utf-8').encode('latin1')
107
- for part in (key_acct, key_cont, key_obj))
108
- else:
109
- alt_path = tuple(
110
- part.encode('latin1').decode('utf-8')
111
- for part in (key_acct, key_cont, key_obj))
103
+ alt_path = tuple(
104
+ part.encode('latin1').decode('utf-8')
105
+ for part in (key_acct, key_cont, key_obj))
112
106
  except UnicodeError:
113
107
  # Well, it was worth a shot
114
108
  pass
@@ -208,10 +202,10 @@ class BaseKeyMaster(object):
208
202
 
209
203
  This provides some basic helpers for:
210
204
 
211
- - loading from a separate config path,
212
- - deriving keys based on path, and
213
- - installing a ``swift.callback.fetch_crypto_keys`` hook
214
- in the request environment.
205
+ - loading from a separate config path,
206
+ - deriving keys based on path, and
207
+ - installing a ``swift.callback.fetch_crypto_keys`` hook
208
+ in the request environment.
215
209
 
216
210
  Subclasses should define ``log_route``, ``keymaster_opts``, and
217
211
  ``keymaster_conf_section`` attributes, and implement the
@@ -336,8 +330,7 @@ class BaseKeyMaster(object):
336
330
  self.logger.warning('Unrecognised secret id: %s' % secret_id)
337
331
  raise UnknownSecretIdError(secret_id)
338
332
  else:
339
- if not six.PY2:
340
- path = path.encode('utf-8')
333
+ path = path.encode('utf-8')
341
334
  return hmac.new(key, path, digestmod=hashlib.sha256).digest()
342
335
 
343
336
 
@@ -34,7 +34,7 @@ class KmsKeyMaster(BaseKeyMaster):
34
34
  'domain_id', 'domain_name', 'project_id',
35
35
  'project_domain_id', 'reauthenticate',
36
36
  'auth_endpoint', 'api_class', 'key_id*',
37
- 'active_root_secret_id')
37
+ 'barbican_endpoint', 'active_root_secret_id')
38
38
  keymaster_conf_section = 'kms_keymaster'
39
39
 
40
40
  def _get_root_secret(self, conf):
@@ -67,6 +67,7 @@ class KmsKeyMaster(BaseKeyMaster):
67
67
  oslo_conf = cfg.ConfigOpts()
68
68
  options.set_defaults(
69
69
  oslo_conf, auth_endpoint=conf.get('auth_endpoint'),
70
+ barbican_endpoint=conf.get('barbican_endpoint'),
70
71
  api_class=conf.get('api_class')
71
72
  )
72
73
  options.enable_logging()
@@ -120,18 +120,17 @@ Here's an example using ``curl`` with tiny 1-byte segments::
120
120
 
121
121
  import json
122
122
 
123
- import six
124
-
125
- from hashlib import md5
126
123
  from swift.common import constraints
127
124
  from swift.common.exceptions import ListingIterError, SegmentError
128
125
  from swift.common.http import is_success
129
- from swift.common.swob import Request, Response, \
126
+ from swift.common.swob import Request, Response, HTTPException, \
130
127
  HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
131
- str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
128
+ str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote, normalize_etag
132
129
  from swift.common.utils import get_logger, \
133
- RateLimitedIterator, quote, close_if_possible, closing_if_possible
134
- from swift.common.request_helpers import SegmentedIterable
130
+ RateLimitedIterator, quote, close_if_possible, closing_if_possible, \
131
+ drain_and_close, md5
132
+ from swift.common.request_helpers import SegmentedIterable, \
133
+ update_ignore_range_header
135
134
  from swift.common.wsgi import WSGIContext, make_subrequest, load_app_config
136
135
 
137
136
 
@@ -206,8 +205,6 @@ class GetContext(WSGIContext):
206
205
  break
207
206
 
208
207
  seg_name = segment['name']
209
- if six.PY2:
210
- seg_name = seg_name.encode("utf-8")
211
208
 
212
209
  # We deliberately omit the etag and size here;
213
210
  # SegmentedIterable will check size and etag if
@@ -331,9 +328,9 @@ class GetContext(WSGIContext):
331
328
  if have_complete_listing:
332
329
  response_headers = [(h, v) for h, v in response_headers
333
330
  if h.lower() != "etag"]
334
- etag = md5()
331
+ etag = md5(usedforsecurity=False)
335
332
  for seg_dict in segments:
336
- etag.update(seg_dict['hash'].strip('"').encode('utf8'))
333
+ etag.update(normalize_etag(seg_dict['hash']).encode('utf8'))
337
334
  response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
338
335
 
339
336
  app_iter = None
@@ -353,6 +350,8 @@ class GetContext(WSGIContext):
353
350
 
354
351
  try:
355
352
  app_iter.validate_first_segment()
353
+ except HTTPException as err_resp:
354
+ return err_resp
356
355
  except (SegmentError, ListingIterError):
357
356
  return HTTPConflict(request=req)
358
357
 
@@ -369,11 +368,16 @@ class GetContext(WSGIContext):
369
368
 
370
369
  Otherwise, simply pass it through.
371
370
  """
371
+ update_ignore_range_header(req, 'X-Object-Manifest')
372
372
  resp_iter = self._app_call(req.environ)
373
373
 
374
374
  # make sure this response is for a dynamic large object manifest
375
375
  for header, value in self._response_headers:
376
376
  if (header.lower() == 'x-object-manifest'):
377
+ content_length = self._response_header_value('content-length')
378
+ if content_length is not None and int(content_length) < 1024:
379
+ # Go ahead and consume small bodies
380
+ drain_and_close(resp_iter)
377
381
  close_if_possible(resp_iter)
378
382
  response = self.get_or_head_response(
379
383
  req, wsgi_to_str(wsgi_unquote(value)))
@@ -99,9 +99,9 @@ storage end points as sync destinations.
99
99
  """
100
100
 
101
101
  from swift.common.middleware import RewriteContext
102
- from swift.common.swob import Request, HTTPBadRequest
103
- from swift.common.utils import config_true_value, list_from_csv, \
104
- register_swift_info
102
+ from swift.common.swob import Request, HTTPBadRequest, wsgi_quote
103
+ from swift.common.utils import config_true_value, list_from_csv
104
+ from swift.common.registry import register_swift_info
105
105
 
106
106
 
107
107
  class _DomainRemapContext(RewriteContext):
@@ -192,7 +192,8 @@ class DomainRemapMiddleware(object):
192
192
  new_path = '/'.join(new_path_parts)
193
193
  env['PATH_INFO'] = new_path
194
194
 
195
- context = _DomainRemapContext(self.app, requested_path, new_path)
195
+ context = _DomainRemapContext(
196
+ self.app, wsgi_quote(requested_path), wsgi_quote(new_path))
196
197
  return context.handle_request(env, start_response)
197
198
 
198
199
  return self.app(env, start_response)
@@ -0,0 +1,128 @@
1
+ # Copyright (c) 2010-2020 OpenStack Foundation
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
+ # implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """
17
+ This middleware fix the Etag header of responses so that it is RFC compliant.
18
+ `RFC 7232 <https://tools.ietf.org/html/rfc7232#section-2.3>`__ specifies that
19
+ the value of the Etag header must be double quoted.
20
+
21
+ It must be placed at the beggining of the pipeline, right after cache::
22
+
23
+ [pipeline:main]
24
+ pipeline = ... cache etag-quoter ...
25
+
26
+ [filter:etag-quoter]
27
+ use = egg:swift#etag_quoter
28
+
29
+ Set ``X-Account-Rfc-Compliant-Etags: true`` at the account
30
+ level to have any Etags in object responses be double quoted, as in
31
+ ``"d41d8cd98f00b204e9800998ecf8427e"``. Alternatively, you may
32
+ only fix Etags in a single container by setting
33
+ ``X-Container-Rfc-Compliant-Etags: true`` on the container.
34
+ This may be necessary for Swift to work properly with some CDNs.
35
+
36
+ Either option may also be explicitly *disabled*, so you may enable quoted
37
+ Etags account-wide as above but turn them off for individual containers
38
+ with ``X-Container-Rfc-Compliant-Etags: false``. This may be
39
+ useful if some subset of applications expect Etags to be bare MD5s.
40
+ """
41
+
42
+ from swift.common.constraints import valid_api_version
43
+ from swift.common.http import is_success
44
+ from swift.common.swob import Request
45
+ from swift.common.utils import config_true_value
46
+ from swift.common.registry import register_swift_info
47
+ from swift.proxy.controllers.base import get_account_info, get_container_info
48
+
49
+
50
+ class EtagQuoterMiddleware(object):
51
+ def __init__(self, app, conf):
52
+ self.app = app
53
+ self.conf = conf
54
+
55
+ def __call__(self, env, start_response):
56
+ req = Request(env)
57
+ try:
58
+ version, account, container, obj = req.split_path(
59
+ 2, 4, rest_with_last=True)
60
+ is_swifty_request = valid_api_version(version)
61
+ except ValueError:
62
+ is_swifty_request = False
63
+
64
+ if not is_swifty_request:
65
+ return self.app(env, start_response)
66
+
67
+ if not obj:
68
+ typ = 'Container' if container else 'Account'
69
+ client_header = 'X-%s-Rfc-Compliant-Etags' % typ
70
+ sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ
71
+ if client_header in req.headers:
72
+ if req.headers[client_header]:
73
+ req.headers[sysmeta_header] = config_true_value(
74
+ req.headers[client_header])
75
+ else:
76
+ req.headers[sysmeta_header] = ''
77
+ if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)):
78
+ req.headers[sysmeta_header] = ''
79
+
80
+ def translating_start_response(status, headers, exc_info=None):
81
+ return start_response(status, [
82
+ (client_header if h.title() == sysmeta_header else h,
83
+ v) for h, v in headers
84
+ ], exc_info)
85
+
86
+ return self.app(env, translating_start_response)
87
+
88
+ container_info = get_container_info(env, self.app, 'EQ')
89
+ if not container_info or not is_success(container_info['status']):
90
+ return self.app(env, start_response)
91
+
92
+ flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags')
93
+ if flag is None:
94
+ account_info = get_account_info(env, self.app, 'EQ')
95
+ if not account_info or not is_success(account_info['status']):
96
+ return self.app(env, start_response)
97
+
98
+ flag = account_info.get('sysmeta', {}).get(
99
+ 'rfc-compliant-etags')
100
+
101
+ if flag is None:
102
+ flag = self.conf.get('enable_by_default', 'false')
103
+
104
+ if not config_true_value(flag):
105
+ return self.app(env, start_response)
106
+
107
+ status, headers, resp_iter = req.call_application(self.app)
108
+
109
+ headers = [
110
+ (header, value) if header.lower() != 'etag' or (
111
+ value.startswith(('"', 'W/"')) and value.endswith('"'))
112
+ else (header, '"%s"' % value)
113
+ for header, value in headers]
114
+
115
+ start_response(status, headers)
116
+ return resp_iter
117
+
118
+
119
+ def filter_factory(global_conf, **local_conf):
120
+ conf = global_conf.copy()
121
+ conf.update(local_conf)
122
+ register_swift_info(
123
+ 'etag_quoter', enable_by_default=config_true_value(
124
+ conf.get('enable_by_default', 'false')))
125
+
126
+ def etag_quoter_filter(app):
127
+ return EtagQuoterMiddleware(app, conf)
128
+ return etag_quoter_filter
@@ -84,11 +84,11 @@ desired.
84
84
  The expires attribute is the Unix timestamp before which the form
85
85
  must be submitted before it is invalidated.
86
86
 
87
- The signature attribute is the HMAC-SHA1 signature of the form. Here is
87
+ The signature attribute is the HMAC signature of the form. Here is
88
88
  sample code for computing the signature::
89
89
 
90
90
  import hmac
91
- from hashlib import sha1
91
+ from hashlib import sha512
92
92
  from time import time
93
93
  path = '/v1/account/container/object_prefix'
94
94
  redirect = 'https://srv.com/some-page' # set to '' if redirect not in form
@@ -98,7 +98,7 @@ sample code for computing the signature::
98
98
  key = 'mykey'
99
99
  hmac_body = '%s\n%s\n%s\n%s\n%s' % (path, redirect,
100
100
  max_file_size, max_file_count, expires)
101
- signature = hmac.new(key, hmac_body, sha1).hexdigest()
101
+ signature = hmac.new(key, hmac_body, sha512).hexdigest()
102
102
 
103
103
  The key is the value of either the account (X-Account-Meta-Temp-URL-Key,
104
104
  X-Account-Meta-Temp-Url-Key-2) or the container
@@ -123,20 +123,22 @@ the file are simply ignored).
123
123
  __all__ = ['FormPost', 'filter_factory', 'READ_CHUNK_SIZE', 'MAX_VALUE_LENGTH']
124
124
 
125
125
  import hmac
126
- from hashlib import sha1
127
126
  from time import time
128
127
 
129
- import six
130
- from six.moves.urllib.parse import quote
128
+ from urllib.parse import quote
131
129
 
132
130
  from swift.common.constraints import valid_api_version
133
131
  from swift.common.exceptions import MimeInvalid
134
132
  from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
135
- from swift.common.utils import streq_const_time, register_swift_info, \
136
- parse_content_disposition, parse_mime_headers, \
137
- iter_multipart_mime_documents, reiterate, close_if_possible
138
- from swift.common.wsgi import make_pre_authed_env
133
+ from swift.common.digest import get_allowed_digests, \
134
+ extract_digest_and_algorithm, DEFAULT_ALLOWED_DIGESTS
135
+ from swift.common.utils import streq_const_time, parse_content_disposition, \
136
+ parse_mime_headers, iter_multipart_mime_documents, reiterate, \
137
+ closing_if_possible, get_logger, InputProxy
138
+ from swift.common.registry import register_swift_info
139
+ from swift.common.wsgi import WSGIContext, make_pre_authed_env
139
140
  from swift.common.swob import HTTPUnauthorized, wsgi_to_str, str_to_wsgi
141
+ from swift.common.http import is_success
140
142
  from swift.proxy.controllers.base import get_account_info, get_container_info
141
143
 
142
144
 
@@ -156,7 +158,7 @@ class FormUnauthorized(Exception):
156
158
  pass
157
159
 
158
160
 
159
- class _CappedFileLikeObject(object):
161
+ class _CappedFileLikeObject(InputProxy):
160
162
  """
161
163
  A file-like object wrapping another file-like object that raises
162
164
  an EOFError if the amount of data read exceeds a given
@@ -168,26 +170,15 @@ class _CappedFileLikeObject(object):
168
170
  """
169
171
 
170
172
  def __init__(self, fp, max_file_size):
171
- self.fp = fp
173
+ super().__init__(fp)
172
174
  self.max_file_size = max_file_size
173
- self.amount_read = 0
174
175
  self.file_size_exceeded = False
175
176
 
176
- def read(self, size=None):
177
- ret = self.fp.read(size)
178
- self.amount_read += len(ret)
179
- if self.amount_read > self.max_file_size:
177
+ def chunk_update(self, chunk, eof, *args, **kwargs):
178
+ if self.bytes_received > self.max_file_size:
180
179
  self.file_size_exceeded = True
181
180
  raise EOFError('max_file_size exceeded')
182
- return ret
183
-
184
- def readline(self):
185
- ret = self.fp.readline()
186
- self.amount_read += len(ret)
187
- if self.amount_read > self.max_file_size:
188
- self.file_size_exceeded = True
189
- raise EOFError('max_file_size exceeded')
190
- return ret
181
+ return chunk
191
182
 
192
183
 
193
184
  class FormPost(object):
@@ -204,11 +195,17 @@ class FormPost(object):
204
195
  :param conf: The configuration dict for the middleware.
205
196
  """
206
197
 
207
- def __init__(self, app, conf):
198
+ def __init__(self, app, conf, logger=None):
208
199
  #: The next WSGI application/filter in the paste.deploy pipeline.
209
200
  self.app = app
210
201
  #: The filter configuration dict.
211
202
  self.conf = conf
203
+ self.logger = logger or get_logger(conf, log_route='formpost')
204
+ # Defaulting to SUPPORTED_DIGESTS just so we don't completely
205
+ # deprecate sha1 yet. We'll change this to DEFAULT_ALLOWED_DIGESTS
206
+ # later.
207
+ self.allowed_digests = conf.get(
208
+ 'allowed_digests', DEFAULT_ALLOWED_DIGESTS.split())
212
209
 
213
210
  def __call__(self, env, start_response):
214
211
  """
@@ -239,9 +236,7 @@ class FormPost(object):
239
236
  ('Content-Length', str(len(body)))))
240
237
  return [body]
241
238
  except (FormInvalid, EOFError) as err:
242
- body = 'FormPost: %s' % err
243
- if six.PY3:
244
- body = body.encode('utf-8')
239
+ body = ('FormPost: %s' % err).encode('utf-8')
245
240
  start_response(
246
241
  '400 Bad Request',
247
242
  (('Content-Type', 'text/plain'),
@@ -263,11 +258,12 @@ class FormPost(object):
263
258
  :returns: status_line, headers_list, body
264
259
  """
265
260
  keys = self._get_keys(env)
266
- if six.PY3:
267
- boundary = boundary.encode('utf-8')
261
+ boundary = boundary.encode('utf-8')
268
262
  status = message = ''
269
263
  attributes = {}
264
+ file_attributes = {}
270
265
  subheaders = []
266
+ resp_body = None
271
267
  file_count = 0
272
268
  for fp in iter_multipart_mime_documents(
273
269
  env['wsgi.input'], boundary, read_chunk_size=READ_CHUNK_SIZE):
@@ -283,16 +279,19 @@ class FormPost(object):
283
279
  break
284
280
  except ValueError:
285
281
  raise FormInvalid('max_file_count not an integer')
286
- attributes['filename'] = attrs['filename'] or 'filename'
282
+ file_attributes = attributes.copy()
283
+ file_attributes['filename'] = attrs['filename'] or 'filename'
287
284
  if 'content-type' not in attributes and 'content-type' in hdrs:
288
- attributes['content-type'] = \
285
+ file_attributes['content-type'] = \
289
286
  hdrs['Content-Type'] or 'application/octet-stream'
290
287
  if 'content-encoding' not in attributes and \
291
288
  'content-encoding' in hdrs:
292
- attributes['content-encoding'] = hdrs['Content-Encoding']
293
- status, subheaders = \
294
- self._perform_subrequest(env, attributes, fp, keys)
295
- if not status.startswith('2'):
289
+ file_attributes['content-encoding'] = \
290
+ hdrs['Content-Encoding']
291
+ status, subheaders, resp_body = \
292
+ self._perform_subrequest(env, file_attributes, fp, keys)
293
+ status_code = int(status.split(' ', 1)[0])
294
+ if not is_success(status_code):
296
295
  break
297
296
  else:
298
297
  data = b''
@@ -305,14 +304,14 @@ class FormPost(object):
305
304
  data += chunk
306
305
  while fp.read(READ_CHUNK_SIZE):
307
306
  pass
308
- if six.PY3:
309
- data = data.decode('utf-8')
307
+ data = data.decode('utf-8')
310
308
  if 'name' in attrs:
311
309
  attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
312
310
  if not status:
313
311
  status = '400 Bad Request'
314
312
  message = 'no files to process'
315
313
 
314
+ status_code = int(status.split(' ', 1)[0])
316
315
  headers = [(k, v) for k, v in subheaders
317
316
  if k.lower().startswith('access-control')]
318
317
 
@@ -321,21 +320,21 @@ class FormPost(object):
321
320
  body = status
322
321
  if message:
323
322
  body = status + '\r\nFormPost: ' + message.title()
323
+ body = body.encode('utf-8')
324
+ if not is_success(status_code) and resp_body:
325
+ body = resp_body
324
326
  headers.extend([('Content-Type', 'text/plain'),
325
327
  ('Content-Length', len(body))])
326
- if six.PY3:
327
- body = body.encode('utf-8')
328
328
  return status, headers, body
329
- status = status.split(' ', 1)[0]
330
329
  if '?' in redirect:
331
330
  redirect += '&'
332
331
  else:
333
332
  redirect += '?'
334
- redirect += 'status=%s&message=%s' % (quote(status), quote(message))
333
+ redirect += 'status=%s&message=%s' % (quote(str(status_code)),
334
+ quote(message))
335
335
  body = '<html><body><p><a href="%s">' \
336
336
  'Click to continue...</a></p></body></html>' % redirect
337
- if six.PY3:
338
- body = body.encode('utf-8')
337
+ body = body.encode('utf-8')
339
338
  headers.extend(
340
339
  [('Location', redirect), ('Content-Length', str(len(body)))])
341
340
  return '303 See Other', headers, body
@@ -397,37 +396,35 @@ class FormPost(object):
397
396
  attributes.get('max_file_size') or '0',
398
397
  attributes.get('max_file_count') or '0',
399
398
  attributes.get('expires') or '0')
400
- if six.PY3:
401
- hmac_body = hmac_body.encode('utf-8')
399
+ hmac_body = hmac_body.encode('utf-8')
402
400
 
403
401
  has_valid_sig = False
402
+ signature = attributes.get('signature', '')
403
+ try:
404
+ hash_name, signature = extract_digest_and_algorithm(signature)
405
+ except ValueError:
406
+ raise FormUnauthorized('invalid signature')
407
+ if hash_name not in self.allowed_digests:
408
+ raise FormUnauthorized('invalid signature')
409
+
404
410
  for key in keys:
405
411
  # Encode key like in swift.common.utls.get_hmac.
406
- if not isinstance(key, six.binary_type):
412
+ if not isinstance(key, bytes):
407
413
  key = key.encode('utf8')
408
- sig = hmac.new(key, hmac_body, sha1).hexdigest()
409
- if streq_const_time(sig, (attributes.get('signature') or
410
- 'invalid')):
414
+ sig = hmac.new(key, hmac_body, hash_name).hexdigest()
415
+ if streq_const_time(sig, signature):
411
416
  has_valid_sig = True
412
417
  if not has_valid_sig:
413
418
  raise FormUnauthorized('invalid signature')
414
-
415
- substatus = [None]
416
- subheaders = [None]
417
-
419
+ self.logger.increment('formpost.digests.%s' % hash_name)
420
+ wsgi_ctx = WSGIContext(self.app)
418
421
  wsgi_input = subenv['wsgi.input']
419
-
420
- def _start_response(status, headers, exc_info=None):
421
- if wsgi_input.file_size_exceeded:
422
- raise EOFError("max_file_size exceeded")
423
-
424
- substatus[0] = status
425
- subheaders[0] = headers
426
-
427
- # reiterate to ensure the response started,
428
- # but drop any data on the floor
429
- close_if_possible(reiterate(self.app(subenv, _start_response)))
430
- return substatus[0], subheaders[0]
422
+ resp = wsgi_ctx._app_call(subenv)
423
+ if wsgi_input.file_size_exceeded:
424
+ raise EOFError("max_file_size exceeded")
425
+ with closing_if_possible(reiterate(resp)):
426
+ body = b''.join(resp)
427
+ return wsgi_ctx._response_status, wsgi_ctx._response_headers, body
431
428
 
432
429
  def _get_keys(self, env):
433
430
  """
@@ -463,6 +460,12 @@ def filter_factory(global_conf, **local_conf):
463
460
  conf = global_conf.copy()
464
461
  conf.update(local_conf)
465
462
 
466
- register_swift_info('formpost')
467
-
463
+ logger = get_logger(conf, log_route='formpost')
464
+ allowed_digests, deprecated_digests = get_allowed_digests(
465
+ conf.get('allowed_digests', '').split(), logger)
466
+ info = {'allowed_digests': sorted(allowed_digests)}
467
+ if deprecated_digests:
468
+ info['deprecated_digests'] = sorted(deprecated_digests)
469
+ register_swift_info('formpost', **info)
470
+ conf.update(info)
468
471
  return lambda app: FormPost(app, conf)