swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +183 -29
  17. swift/cli/manage_shard_ranges.py +708 -37
  18. swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +198 -127
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +396 -147
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -81
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +52 -19
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +192 -174
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2191 -2762
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +555 -536
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +552 -227
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +397 -176
  135. swift/container/sharder.py +1580 -639
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +213 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +653 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +452 -86
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1009 -490
  156. swift/proxy/server.py +185 -112
  157. swift-2.35.0.dist-info/AUTHORS +501 -0
  158. swift-2.35.0.dist-info/LICENSE +202 -0
  159. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
  160. swift-2.35.0.dist-info/RECORD +201 -0
  161. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  162. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  163. swift-2.35.0.dist-info/pbr.json +1 -0
  164. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  165. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  166. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  167. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  168. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  169. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  170. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  171. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  172. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  173. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  174. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  175. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  176. swift-2.23.2.data/scripts/swift-account-auditor +0 -23
  177. swift-2.23.2.data/scripts/swift-account-info +0 -51
  178. swift-2.23.2.data/scripts/swift-account-reaper +0 -23
  179. swift-2.23.2.data/scripts/swift-account-replicator +0 -34
  180. swift-2.23.2.data/scripts/swift-account-server +0 -23
  181. swift-2.23.2.data/scripts/swift-container-auditor +0 -23
  182. swift-2.23.2.data/scripts/swift-container-info +0 -51
  183. swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
  184. swift-2.23.2.data/scripts/swift-container-replicator +0 -34
  185. swift-2.23.2.data/scripts/swift-container-sharder +0 -33
  186. swift-2.23.2.data/scripts/swift-container-sync +0 -23
  187. swift-2.23.2.data/scripts/swift-container-updater +0 -23
  188. swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
  189. swift-2.23.2.data/scripts/swift-form-signature +0 -20
  190. swift-2.23.2.data/scripts/swift-init +0 -119
  191. swift-2.23.2.data/scripts/swift-object-auditor +0 -29
  192. swift-2.23.2.data/scripts/swift-object-expirer +0 -33
  193. swift-2.23.2.data/scripts/swift-object-info +0 -60
  194. swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
  195. swift-2.23.2.data/scripts/swift-object-relinker +0 -41
  196. swift-2.23.2.data/scripts/swift-object-replicator +0 -37
  197. swift-2.23.2.data/scripts/swift-object-server +0 -27
  198. swift-2.23.2.data/scripts/swift-object-updater +0 -23
  199. swift-2.23.2.data/scripts/swift-proxy-server +0 -23
  200. swift-2.23.2.data/scripts/swift-recon +0 -24
  201. swift-2.23.2.data/scripts/swift-ring-builder +0 -24
  202. swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
  203. swift-2.23.2.data/scripts/swift-ring-composer +0 -22
  204. swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
  205. swift-2.23.2.dist-info/RECORD +0 -220
  206. swift-2.23.2.dist-info/metadata.json +0 -1
  207. swift-2.23.2.dist-info/pbr.json +0 -1
  208. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -16,21 +16,25 @@
16
16
  from base64 import standard_b64encode as b64encode
17
17
  from base64 import standard_b64decode as b64decode
18
18
 
19
- import six
20
- from six.moves.urllib.parse import quote
19
+ from urllib.parse import quote
21
20
 
22
21
  from swift.common import swob
23
22
  from swift.common.http import HTTP_OK
24
- from swift.common.utils import json, public, config_true_value
23
+ from swift.common.middleware.versioned_writes.object_versioning import \
24
+ DELETE_MARKER_CONTENT_TYPE
25
+ from swift.common.utils import json, public, config_true_value, Timestamp, \
26
+ cap_length
27
+ from swift.common.registry import get_swift_info
25
28
 
26
29
  from swift.common.middleware.s3api.controllers.base import Controller
27
- from swift.common.middleware.s3api.etree import Element, SubElement, tostring, \
28
- fromstring, XMLSyntaxError, DocumentInvalid
29
- from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
30
- InvalidArgument, \
30
+ from swift.common.middleware.s3api.etree import Element, SubElement, \
31
+ tostring, fromstring, XMLSyntaxError, DocumentInvalid
32
+ from swift.common.middleware.s3api.s3response import \
33
+ HTTPOk, S3NotImplemented, InvalidArgument, \
31
34
  MalformedXML, InvalidLocationConstraint, NoSuchBucket, \
32
- BucketNotEmpty, InternalError, ServiceUnavailable, NoSuchKey
33
- from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX
35
+ BucketNotEmpty, VersionedBucketNotEmpty, InternalError, \
36
+ ServiceUnavailable, NoSuchKey
37
+ from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, S3Timestamp
34
38
 
35
39
  MAX_PUT_BUCKET_BODY_SIZE = 10240
36
40
 
@@ -50,7 +54,10 @@ class BucketController(Controller):
50
54
  try:
51
55
  resp = req.get_response(self.app, 'HEAD')
52
56
  if int(resp.sw_headers['X-Container-Object-Count']) > 0:
53
- raise BucketNotEmpty()
57
+ if resp.sw_headers.get('X-Container-Sysmeta-Versions-Enabled'):
58
+ raise VersionedBucketNotEmpty()
59
+ else:
60
+ raise BucketNotEmpty()
54
61
  # FIXME: This extra HEAD saves unexpected segment deletion
55
62
  # but if a complete multipart upload happen while cleanup
56
63
  # segment container below, completed object may be missing its
@@ -94,170 +101,266 @@ class BucketController(Controller):
94
101
 
95
102
  return HTTPOk(headers=resp.headers)
96
103
 
97
- @public
98
- def GET(self, req):
99
- """
100
- Handle GET Bucket (List Objects) request
101
- """
102
-
103
- max_keys = req.get_validated_param(
104
- 'max-keys', self.conf.max_bucket_listing)
105
- # TODO: Separate max_bucket_listing and default_bucket_listing
106
- tag_max_keys = max_keys
107
- max_keys = min(max_keys, self.conf.max_bucket_listing)
108
-
104
+ def _parse_request_options(self, req, max_keys):
109
105
  encoding_type = req.params.get('encoding-type')
110
106
  if encoding_type is not None and encoding_type != 'url':
111
107
  err_msg = 'Invalid Encoding Method specified in Request'
112
108
  raise InvalidArgument('encoding-type', encoding_type, err_msg)
113
109
 
110
+ # in order to judge that truncated is valid, check whether
111
+ # max_keys + 1 th element exists in swift.
114
112
  query = {
115
- 'format': 'json',
116
113
  'limit': max_keys + 1,
117
114
  }
118
115
  if 'prefix' in req.params:
119
- query.update({'prefix': req.params['prefix']})
116
+ query['prefix'] = swob.wsgi_to_str(req.params['prefix'])
120
117
  if 'delimiter' in req.params:
121
- query.update({'delimiter': req.params['delimiter']})
118
+ query['delimiter'] = swob.wsgi_to_str(req.params['delimiter'])
122
119
  fetch_owner = False
123
120
  if 'versions' in req.params:
121
+ query['versions'] = swob.wsgi_to_str(req.params['versions'])
124
122
  listing_type = 'object-versions'
123
+ version_marker = swob.wsgi_to_str(req.params.get(
124
+ 'version-id-marker'))
125
125
  if 'key-marker' in req.params:
126
- query.update({'marker': req.params['key-marker']})
127
- elif 'version-id-marker' in req.params:
126
+ query['marker'] = swob.wsgi_to_str(req.params['key-marker'])
127
+ if version_marker is not None:
128
+ if version_marker != 'null':
129
+ try:
130
+ Timestamp(version_marker)
131
+ except ValueError:
132
+ raise InvalidArgument(
133
+ 'version-id-marker', version_marker,
134
+ 'Invalid version id specified')
135
+ query['version_marker'] = version_marker
136
+ elif version_marker is not None:
128
137
  err_msg = ('A version-id marker cannot be specified without '
129
138
  'a key marker.')
130
139
  raise InvalidArgument('version-id-marker',
131
- req.params['version-id-marker'], err_msg)
140
+ version_marker, err_msg)
132
141
  elif int(req.params.get('list-type', '1')) == 2:
133
142
  listing_type = 'version-2'
134
143
  if 'start-after' in req.params:
135
- query.update({'marker': req.params['start-after']})
144
+ query['marker'] = swob.wsgi_to_str(req.params['start-after'])
136
145
  # continuation-token overrides start-after
137
146
  if 'continuation-token' in req.params:
138
- decoded = b64decode(req.params['continuation-token'])
139
- if not six.PY2:
140
- decoded = decoded.decode('utf8')
141
- query.update({'marker': decoded})
147
+ decoded = b64decode(
148
+ req.params['continuation-token']).decode('utf8')
149
+ query['marker'] = decoded
142
150
  if 'fetch-owner' in req.params:
143
151
  fetch_owner = config_true_value(req.params['fetch-owner'])
144
152
  else:
145
153
  listing_type = 'version-1'
146
154
  if 'marker' in req.params:
147
- query.update({'marker': req.params['marker']})
148
-
149
- resp = req.get_response(self.app, query=query)
155
+ query['marker'] = swob.wsgi_to_str(req.params['marker'])
156
+
157
+ return encoding_type, query, listing_type, fetch_owner
158
+
159
+ def _build_versions_result(self, req, objects, encoding_type,
160
+ tag_max_keys, is_truncated):
161
+ elem = Element('ListVersionsResult')
162
+ SubElement(elem, 'Name').text = req.container_name
163
+ prefix = swob.wsgi_to_str(req.params.get('prefix'))
164
+ if prefix and encoding_type == 'url':
165
+ prefix = quote(prefix)
166
+ SubElement(elem, 'Prefix').text = prefix
167
+ key_marker = swob.wsgi_to_str(req.params.get('key-marker'))
168
+ if key_marker and encoding_type == 'url':
169
+ key_marker = quote(key_marker)
170
+ SubElement(elem, 'KeyMarker').text = key_marker
171
+ SubElement(elem, 'VersionIdMarker').text = swob.wsgi_to_str(
172
+ req.params.get('version-id-marker'))
173
+ if is_truncated:
174
+ if 'name' in objects[-1]:
175
+ SubElement(elem, 'NextKeyMarker').text = \
176
+ objects[-1]['name']
177
+ SubElement(elem, 'NextVersionIdMarker').text = \
178
+ objects[-1].get('version') or 'null'
179
+ if 'subdir' in objects[-1]:
180
+ SubElement(elem, 'NextKeyMarker').text = \
181
+ objects[-1]['subdir']
182
+ SubElement(elem, 'NextVersionIdMarker').text = 'null'
183
+ SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
184
+ delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
185
+ if delimiter is not None:
186
+ if encoding_type == 'url':
187
+ delimiter = quote(delimiter)
188
+ SubElement(elem, 'Delimiter').text = delimiter
189
+ if encoding_type == 'url':
190
+ SubElement(elem, 'EncodingType').text = encoding_type
191
+ SubElement(elem, 'IsTruncated').text = \
192
+ 'true' if is_truncated else 'false'
193
+ return elem
194
+
195
+ def _build_base_listing_element(self, req, encoding_type):
196
+ elem = Element('ListBucketResult')
197
+ SubElement(elem, 'Name').text = req.container_name
198
+ prefix = swob.wsgi_to_str(req.params.get('prefix'))
199
+ if prefix and encoding_type == 'url':
200
+ prefix = quote(prefix)
201
+ SubElement(elem, 'Prefix').text = prefix
202
+ return elem
203
+
204
+ def _build_list_bucket_result_type_one(self, req, objects, encoding_type,
205
+ tag_max_keys, is_truncated):
206
+ elem = self._build_base_listing_element(req, encoding_type)
207
+ marker = swob.wsgi_to_str(req.params.get('marker'))
208
+ if marker and encoding_type == 'url':
209
+ marker = quote(marker)
210
+ SubElement(elem, 'Marker').text = marker
211
+ if is_truncated and 'delimiter' in req.params:
212
+ if 'name' in objects[-1]:
213
+ name = objects[-1]['name']
214
+ else:
215
+ name = objects[-1]['subdir']
216
+ if encoding_type == 'url':
217
+ name = quote(name.encode('utf-8'))
218
+ SubElement(elem, 'NextMarker').text = name
219
+ # XXX: really? no NextMarker when no delimiter??
220
+ SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
221
+ delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
222
+ if delimiter:
223
+ if encoding_type == 'url':
224
+ delimiter = quote(delimiter)
225
+ SubElement(elem, 'Delimiter').text = delimiter
226
+ if encoding_type == 'url':
227
+ SubElement(elem, 'EncodingType').text = encoding_type
228
+ SubElement(elem, 'IsTruncated').text = \
229
+ 'true' if is_truncated else 'false'
230
+ return elem
231
+
232
+ def _build_list_bucket_result_type_two(self, req, objects, encoding_type,
233
+ tag_max_keys, is_truncated):
234
+ elem = self._build_base_listing_element(req, encoding_type)
235
+ if is_truncated:
236
+ if 'name' in objects[-1]:
237
+ SubElement(elem, 'NextContinuationToken').text = \
238
+ b64encode(objects[-1]['name'].encode('utf8'))
239
+ if 'subdir' in objects[-1]:
240
+ SubElement(elem, 'NextContinuationToken').text = \
241
+ b64encode(objects[-1]['subdir'].encode('utf8'))
242
+ if 'continuation-token' in req.params:
243
+ SubElement(elem, 'ContinuationToken').text = \
244
+ swob.wsgi_to_str(req.params['continuation-token'])
245
+ start_after = swob.wsgi_to_str(req.params.get('start-after'))
246
+ if start_after is not None:
247
+ if encoding_type == 'url':
248
+ start_after = quote(start_after)
249
+ SubElement(elem, 'StartAfter').text = start_after
250
+ SubElement(elem, 'KeyCount').text = str(len(objects))
251
+ SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
252
+ delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
253
+ if delimiter:
254
+ if encoding_type == 'url':
255
+ delimiter = quote(delimiter)
256
+ SubElement(elem, 'Delimiter').text = delimiter
257
+ if encoding_type == 'url':
258
+ SubElement(elem, 'EncodingType').text = encoding_type
259
+ SubElement(elem, 'IsTruncated').text = \
260
+ 'true' if is_truncated else 'false'
261
+ return elem
150
262
 
151
- objects = json.loads(resp.body)
263
+ def _add_subdir(self, elem, o, encoding_type):
264
+ common_prefixes = SubElement(elem, 'CommonPrefixes')
265
+ name = o['subdir']
266
+ if encoding_type == 'url':
267
+ name = quote(name.encode('utf-8'))
268
+ SubElement(common_prefixes, 'Prefix').text = name
152
269
 
153
- # in order to judge that truncated is valid, check whether
154
- # max_keys + 1 th element exists in swift.
155
- is_truncated = max_keys > 0 and len(objects) > max_keys
156
- objects = objects[:max_keys]
270
+ def _add_object(self, req, elem, o, encoding_type, listing_type,
271
+ fetch_owner):
272
+ name = o['name']
273
+ if encoding_type == 'url':
274
+ name = quote(name.encode('utf-8'))
157
275
 
158
276
  if listing_type == 'object-versions':
159
- elem = Element('ListVersionsResult')
160
- SubElement(elem, 'Name').text = req.container_name
161
- SubElement(elem, 'Prefix').text = req.params.get('prefix')
162
- SubElement(elem, 'KeyMarker').text = req.params.get('key-marker')
163
- SubElement(elem, 'VersionIdMarker').text = req.params.get(
164
- 'version-id-marker')
165
- if is_truncated:
166
- if 'name' in objects[-1]:
167
- SubElement(elem, 'NextKeyMarker').text = \
168
- objects[-1]['name']
169
- if 'subdir' in objects[-1]:
170
- SubElement(elem, 'NextKeyMarker').text = \
171
- objects[-1]['subdir']
172
- SubElement(elem, 'NextVersionIdMarker').text = 'null'
277
+ if o['content_type'] == DELETE_MARKER_CONTENT_TYPE:
278
+ contents = SubElement(elem, 'DeleteMarker')
279
+ else:
280
+ contents = SubElement(elem, 'Version')
281
+ SubElement(contents, 'Key').text = name
282
+ SubElement(contents, 'VersionId').text = o.get(
283
+ 'version_id') or 'null'
284
+ if 'object_versioning' in get_swift_info():
285
+ SubElement(contents, 'IsLatest').text = (
286
+ 'true' if o['is_latest'] else 'false')
287
+ else:
288
+ SubElement(contents, 'IsLatest').text = 'true'
173
289
  else:
174
- elem = Element('ListBucketResult')
175
- SubElement(elem, 'Name').text = req.container_name
176
- SubElement(elem, 'Prefix').text = req.params.get('prefix')
177
- if listing_type == 'version-1':
178
- SubElement(elem, 'Marker').text = req.params.get('marker')
179
- if is_truncated and 'delimiter' in req.params:
180
- if 'name' in objects[-1]:
181
- name = objects[-1]['name']
182
- else:
183
- name = objects[-1]['subdir']
184
- if encoding_type == 'url':
185
- name = quote(name.encode('utf-8'))
186
- SubElement(elem, 'NextMarker').text = name
187
- elif listing_type == 'version-2':
188
- if is_truncated:
189
- if 'name' in objects[-1]:
190
- SubElement(elem, 'NextContinuationToken').text = \
191
- b64encode(objects[-1]['name'].encode('utf-8'))
192
- if 'subdir' in objects[-1]:
193
- SubElement(elem, 'NextContinuationToken').text = \
194
- b64encode(objects[-1]['subdir'].encode('utf-8'))
195
- if 'continuation-token' in req.params:
196
- SubElement(elem, 'ContinuationToken').text = \
197
- req.params['continuation-token']
198
- if 'start-after' in req.params:
199
- SubElement(elem, 'StartAfter').text = \
200
- req.params['start-after']
201
- SubElement(elem, 'KeyCount').text = str(len(objects))
290
+ contents = SubElement(elem, 'Contents')
291
+ SubElement(contents, 'Key').text = name
292
+ SubElement(contents, 'LastModified').text = \
293
+ S3Timestamp.from_isoformat(o['last_modified']).s3xmlformat
294
+ if contents.tag != 'DeleteMarker':
295
+ if 's3_etag' in o:
296
+ # New-enough MUs are already in the right format
297
+ etag = o['s3_etag']
298
+ elif 'slo_etag' in o:
299
+ # SLOs may be in something *close* to the MU format
300
+ etag = '"%s-N"' % o['slo_etag'].strip('"')
301
+ else:
302
+ # Normal objects just use the MD5
303
+ etag = o['hash']
304
+ if len(etag) < 2 or etag[::len(etag) - 1] != '""':
305
+ # Normal objects just use the MD5
306
+ etag = '"%s"' % o['hash']
307
+ # This also catches sufficiently-old SLOs, but we have
308
+ # no way to identify those from container listings
309
+ # Otherwise, somebody somewhere (proxyfs, maybe?) made this
310
+ # look like an RFC-compliant ETag; we don't need to
311
+ # quote-wrap.
312
+ SubElement(contents, 'ETag').text = etag
313
+ SubElement(contents, 'Size').text = str(o['bytes'])
314
+ if fetch_owner or listing_type != 'version-2':
315
+ owner = SubElement(contents, 'Owner')
316
+ SubElement(owner, 'ID').text = req.user_id
317
+ SubElement(owner, 'DisplayName').text = req.user_id
318
+ if contents.tag != 'DeleteMarker':
319
+ SubElement(contents, 'StorageClass').text = 'STANDARD'
320
+
321
+ def _add_objects_to_result(self, req, elem, objects, encoding_type,
322
+ listing_type, fetch_owner):
323
+ for o in objects:
324
+ if 'subdir' in o:
325
+ self._add_subdir(elem, o, encoding_type)
326
+ else:
327
+ self._add_object(req, elem, o, encoding_type, listing_type,
328
+ fetch_owner)
202
329
 
203
- SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
330
+ @public
331
+ def GET(self, req):
332
+ """
333
+ Handle GET Bucket (List Objects) request
334
+ """
335
+ tag_max_keys = req.get_validated_param(
336
+ 'max-keys', self.conf.max_bucket_listing)
337
+ # TODO: Separate max_bucket_listing and default_bucket_listing
338
+ max_keys = min(tag_max_keys, self.conf.max_bucket_listing)
204
339
 
205
- if 'delimiter' in req.params:
206
- SubElement(elem, 'Delimiter').text = req.params['delimiter']
340
+ encoding_type, query, listing_type, fetch_owner = \
341
+ self._parse_request_options(req, max_keys)
207
342
 
208
- if encoding_type == 'url':
209
- SubElement(elem, 'EncodingType').text = encoding_type
343
+ resp = req.get_response(self.app, query=query)
210
344
 
211
- SubElement(elem, 'IsTruncated').text = \
212
- 'true' if is_truncated else 'false'
345
+ try:
346
+ objects = json.loads(resp.body)
347
+ except (TypeError, ValueError):
348
+ self.logger.error('Got non-JSON response trying to list %s: %r',
349
+ req.path, cap_length(resp.body, 60))
350
+ raise
213
351
 
214
- for o in objects:
215
- if 'subdir' not in o:
216
- name = o['name']
217
- if encoding_type == 'url':
218
- name = quote(name.encode('utf-8'))
219
-
220
- if listing_type == 'object-versions':
221
- contents = SubElement(elem, 'Version')
222
- SubElement(contents, 'Key').text = name
223
- SubElement(contents, 'VersionId').text = 'null'
224
- SubElement(contents, 'IsLatest').text = 'true'
225
- else:
226
- contents = SubElement(elem, 'Contents')
227
- SubElement(contents, 'Key').text = name
228
- SubElement(contents, 'LastModified').text = \
229
- o['last_modified'][:-3] + 'Z'
230
- if 's3_etag' in o:
231
- # New-enough MUs are already in the right format
232
- etag = o['s3_etag']
233
- elif 'slo_etag' in o:
234
- # SLOs may be in something *close* to the MU format
235
- etag = '"%s-N"' % o['slo_etag'].strip('"')
236
- else:
237
- etag = o['hash']
238
- if len(etag) < 2 or etag[::len(etag) - 1] != '""':
239
- # Normal objects just use the MD5
240
- etag = '"%s"' % o['hash']
241
- # This also catches sufficiently-old SLOs, but we have
242
- # no way to identify those from container listings
243
- # Otherwise, somebody somewhere (proxyfs, maybe?) made this
244
- # look like an RFC-compliant ETag; we don't need to
245
- # quote-wrap.
246
- SubElement(contents, 'ETag').text = etag
247
- SubElement(contents, 'Size').text = str(o['bytes'])
248
- if fetch_owner or listing_type != 'version-2':
249
- owner = SubElement(contents, 'Owner')
250
- SubElement(owner, 'ID').text = req.user_id
251
- SubElement(owner, 'DisplayName').text = req.user_id
252
- SubElement(contents, 'StorageClass').text = 'STANDARD'
352
+ is_truncated = max_keys > 0 and len(objects) > max_keys
353
+ objects = objects[:max_keys]
253
354
 
254
- for o in objects:
255
- if 'subdir' in o:
256
- common_prefixes = SubElement(elem, 'CommonPrefixes')
257
- name = o['subdir']
258
- if encoding_type == 'url':
259
- name = quote(name.encode('utf-8'))
260
- SubElement(common_prefixes, 'Prefix').text = name
355
+ if listing_type == 'object-versions':
356
+ func = self._build_versions_result
357
+ elif listing_type == 'version-2':
358
+ func = self._build_list_bucket_result_type_two
359
+ else:
360
+ func = self._build_list_bucket_result_type_one
361
+ elem = func(req, objects, encoding_type, tag_max_keys, is_truncated)
362
+ self._add_objects_to_result(
363
+ req, elem, objects, encoding_type, listing_type, fetch_owner)
261
364
 
262
365
  body = tostring(elem)
263
366
 
@@ -281,7 +384,8 @@ class BucketController(Controller):
281
384
  self.logger.error(e)
282
385
  raise
283
386
 
284
- if location != self.conf.location:
387
+ if location not in (self.conf.location,
388
+ self.conf.location.lower()):
285
389
  # s3api cannot support multiple regions currently.
286
390
  raise InvalidLocationConstraint()
287
391
 
@@ -297,6 +401,7 @@ class BucketController(Controller):
297
401
  """
298
402
  Handle DELETE Bucket request
299
403
  """
404
+ # NB: object_versioning is responsible for cleaning up its container
300
405
  if self.conf.allow_multipart_uploads:
301
406
  self._delete_segments_bucket(req)
302
407
  resp = req.get_response(self.app)
@@ -18,8 +18,8 @@ from swift.common.utils import public
18
18
  from swift.common.middleware.s3api.controllers.base import Controller, \
19
19
  bucket_operation
20
20
  from swift.common.middleware.s3api.etree import Element, tostring
21
- from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
22
- NoLoggingStatusForKey
21
+ from swift.common.middleware.s3api.s3response import (
22
+ HTTPOk, S3NotImplemented, NoLoggingStatusForKey)
23
23
 
24
24
 
25
25
  class LoggingStatusController(Controller):
@@ -17,15 +17,18 @@ import copy
17
17
  import json
18
18
 
19
19
  from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
20
+ from swift.common.http import HTTP_NO_CONTENT
21
+ from swift.common.swob import str_to_wsgi
20
22
  from swift.common.utils import public, StreamingPile
23
+ from swift.common.registry import get_swift_info
21
24
 
22
25
  from swift.common.middleware.s3api.controllers.base import Controller, \
23
26
  bucket_operation
24
27
  from swift.common.middleware.s3api.etree import Element, SubElement, \
25
28
  fromstring, tostring, XMLSyntaxError, DocumentInvalid
26
- from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
27
- NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \
28
- AccessDenied, MissingRequestBodyError
29
+ from swift.common.middleware.s3api.s3response import HTTPOk, \
30
+ S3NotImplemented, NoSuchKey, ErrorResponse, MalformedXML, \
31
+ UserKeyMustBeSpecified, AccessDenied, MissingRequestBodyError
29
32
 
30
33
 
31
34
  class MultiObjectDeleteController(Controller):
@@ -35,12 +38,10 @@ class MultiObjectDeleteController(Controller):
35
38
  """
36
39
  def _gen_error_body(self, error, elem, delete_list):
37
40
  for key, version in delete_list:
38
- if version is not None:
39
- # TODO: delete the specific version of the object
40
- raise S3NotImplemented()
41
-
42
41
  error_elem = SubElement(elem, 'Error')
43
42
  SubElement(error_elem, 'Key').text = key
43
+ if version is not None:
44
+ SubElement(error_elem, 'VersionId').text = version
44
45
  SubElement(error_elem, 'Code').text = error.__class__.__name__
45
46
  SubElement(error_elem, 'Message').text = error._msg
46
47
 
@@ -76,14 +77,16 @@ class MultiObjectDeleteController(Controller):
76
77
  if not xml:
77
78
  raise MissingRequestBodyError()
78
79
 
79
- req.check_md5(xml)
80
+ if 'x-amz-content-sha256' not in req.headers:
81
+ # SHA256 got checked when we read the body, so there's at
82
+ # least *some* verification. Recent versions of boto3 stopped
83
+ # sending Content-MD5, so it can't *always* be required.
84
+ # See https://bugs.launchpad.net/swift/+bug/2098529
85
+ req.check_md5(xml)
80
86
  elem = fromstring(xml, 'Delete', self.logger)
81
87
 
82
88
  quiet = elem.find('./Quiet')
83
- if quiet is not None and quiet.text.lower() == 'true':
84
- self.quiet = True
85
- else:
86
- self.quiet = False
89
+ self.quiet = quiet is not None and quiet.text.lower() == 'true'
87
90
 
88
91
  delete_list = list(object_key_iter(elem))
89
92
  if len(delete_list) > self.conf.max_multi_delete_objects:
@@ -105,21 +108,35 @@ class MultiObjectDeleteController(Controller):
105
108
  body = self._gen_error_body(error, elem, delete_list)
106
109
  return HTTPOk(body=body)
107
110
 
108
- if any(version is not None for _key, version in delete_list):
109
- # TODO: support deleting specific versions of objects
111
+ if 'object_versioning' not in get_swift_info() and any(
112
+ version not in ('null', None)
113
+ for _key, version in delete_list):
110
114
  raise S3NotImplemented()
111
115
 
112
116
  def do_delete(base_req, key, version):
113
117
  req = copy.copy(base_req)
114
118
  req.environ = copy.copy(base_req.environ)
115
- req.object_name = key
119
+ req.object_name = str_to_wsgi(key)
120
+ if version:
121
+ req.params = {'version-id': version, 'symlink': 'get'}
116
122
 
117
123
  try:
118
- query = req.gen_multipart_manifest_delete_query(self.app)
124
+ try:
125
+ query = req.gen_multipart_manifest_delete_query(
126
+ self.app, version=version)
127
+ except NoSuchKey:
128
+ query = {}
129
+ if version:
130
+ query['version-id'] = version
131
+ query['symlink'] = 'get'
132
+
119
133
  resp = req.get_response(self.app, method='DELETE', query=query,
120
134
  headers={'Accept': 'application/json'})
121
- # Have to read the response to actually do the SLO delete
122
- if query:
135
+ # If async segment cleanup is available, we expect to get
136
+ # back a 204; otherwise, the delete is synchronous and we
137
+ # have to read the response to actually do the SLO delete
138
+ if query.get('multipart-manifest') and \
139
+ resp.status_int != HTTP_NO_CONTENT:
123
140
  try:
124
141
  delete_result = json.loads(resp.body)
125
142
  if delete_result['Errors']:
@@ -135,8 +152,8 @@ class MultiObjectDeleteController(Controller):
135
152
  except (ValueError, TypeError, KeyError):
136
153
  # Logs get all the gory details
137
154
  self.logger.exception(
138
- 'Could not parse SLO delete response: %r',
139
- resp.body)
155
+ 'Could not parse SLO delete response (%s): %s',
156
+ resp.status, resp.body)
140
157
  # Client gets something more generic
141
158
  return key, {'code': 'SLODeleteError',
142
159
  'message': 'Unexpected swift response'}
@@ -144,6 +161,12 @@ class MultiObjectDeleteController(Controller):
144
161
  pass
145
162
  except ErrorResponse as e:
146
163
  return key, {'code': e.__class__.__name__, 'message': e._msg}
164
+ except Exception:
165
+ self.logger.exception(
166
+ 'Unexpected Error handling DELETE of %r %r' % (
167
+ req.container_name, key))
168
+ return key, {'code': 'Server Error', 'message': 'Server Error'}
169
+
147
170
  return key, None
148
171
 
149
172
  with StreamingPile(self.conf.multi_delete_concurrency) as pile: