swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/__init__.py +29 -50
- swift/account/auditor.py +21 -118
- swift/account/backend.py +33 -28
- swift/account/reaper.py +37 -28
- swift/account/replicator.py +22 -0
- swift/account/server.py +60 -26
- swift/account/utils.py +28 -11
- swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
- swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
- swift/cli/container_deleter.py +5 -11
- swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
- swift/cli/dispersion_report.py +10 -9
- swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
- swift/cli/form_signature.py +3 -7
- swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
- swift/cli/info.py +183 -29
- swift/cli/manage_shard_ranges.py +708 -37
- swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
- swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
- swift/cli/recon.py +196 -67
- swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
- swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +807 -126
- swift/cli/reload.py +135 -0
- swift/cli/ringbuilder.py +217 -20
- swift/cli/ringcomposer.py +0 -1
- swift/cli/shard-info.py +4 -3
- swift/common/base_storage_server.py +9 -20
- swift/common/bufferedhttp.py +48 -74
- swift/common/constraints.py +20 -15
- swift/common/container_sync_realms.py +9 -11
- swift/common/daemon.py +25 -8
- swift/common/db.py +198 -127
- swift/common/db_auditor.py +168 -0
- swift/common/db_replicator.py +95 -55
- swift/common/digest.py +141 -0
- swift/common/direct_client.py +144 -33
- swift/common/error_limiter.py +93 -0
- swift/common/exceptions.py +25 -1
- swift/common/header_key_dict.py +2 -9
- swift/common/http_protocol.py +373 -0
- swift/common/internal_client.py +129 -59
- swift/common/linkat.py +3 -4
- swift/common/manager.py +284 -67
- swift/common/memcached.py +396 -147
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +211 -46
- swift/common/middleware/acl.py +3 -8
- swift/common/middleware/backend_ratelimit.py +230 -0
- swift/common/middleware/bulk.py +22 -34
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +6 -11
- swift/common/middleware/container_quotas.py +1 -1
- swift/common/middleware/container_sync.py +39 -17
- swift/common/middleware/copy.py +12 -0
- swift/common/middleware/crossdomain.py +22 -9
- swift/common/middleware/crypto/__init__.py +2 -1
- swift/common/middleware/crypto/crypto_utils.py +11 -15
- swift/common/middleware/crypto/decrypter.py +28 -11
- swift/common/middleware/crypto/encrypter.py +12 -17
- swift/common/middleware/crypto/keymaster.py +8 -15
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/dlo.py +15 -11
- swift/common/middleware/domain_remap.py +5 -4
- swift/common/middleware/etag_quoter.py +128 -0
- swift/common/middleware/formpost.py +73 -70
- swift/common/middleware/gatekeeper.py +8 -1
- swift/common/middleware/keystoneauth.py +33 -3
- swift/common/middleware/list_endpoints.py +4 -4
- swift/common/middleware/listing_formats.py +85 -49
- swift/common/middleware/memcache.py +4 -81
- swift/common/middleware/name_check.py +3 -2
- swift/common/middleware/proxy_logging.py +160 -92
- swift/common/middleware/ratelimit.py +17 -10
- swift/common/middleware/read_only.py +6 -4
- swift/common/middleware/recon.py +59 -22
- swift/common/middleware/s3api/acl_handlers.py +25 -3
- swift/common/middleware/s3api/acl_utils.py +6 -1
- swift/common/middleware/s3api/controllers/__init__.py +6 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/bucket.py +242 -137
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
- swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
- swift/common/middleware/s3api/controllers/obj.py +112 -8
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
- swift/common/middleware/s3api/controllers/tagging.py +57 -0
- swift/common/middleware/s3api/controllers/versioning.py +36 -7
- swift/common/middleware/s3api/etree.py +22 -9
- swift/common/middleware/s3api/exception.py +0 -4
- swift/common/middleware/s3api/s3api.py +113 -41
- swift/common/middleware/s3api/s3request.py +384 -218
- swift/common/middleware/s3api/s3response.py +126 -23
- swift/common/middleware/s3api/s3token.py +16 -17
- swift/common/middleware/s3api/schema/delete.rng +1 -1
- swift/common/middleware/s3api/subresource.py +7 -10
- swift/common/middleware/s3api/utils.py +27 -10
- swift/common/middleware/slo.py +665 -358
- swift/common/middleware/staticweb.py +64 -37
- swift/common/middleware/symlink.py +52 -19
- swift/common/middleware/tempauth.py +76 -58
- swift/common/middleware/tempurl.py +192 -174
- swift/common/middleware/versioned_writes/__init__.py +51 -0
- swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
- swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +18 -19
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +10 -10
- swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
- swift/common/registry.py +147 -0
- swift/common/request_helpers.py +324 -57
- swift/common/ring/builder.py +67 -25
- swift/common/ring/composite_builder.py +1 -1
- swift/common/ring/ring.py +177 -51
- swift/common/ring/utils.py +1 -1
- swift/common/splice.py +10 -6
- swift/common/statsd_client.py +205 -0
- swift/common/storage_policy.py +49 -44
- swift/common/swob.py +86 -102
- swift/common/{utils.py → utils/__init__.py} +2191 -2762
- swift/common/utils/base.py +131 -0
- swift/common/utils/config.py +433 -0
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/logs.py +859 -0
- swift/common/utils/timestamp.py +412 -0
- swift/common/wsgi.py +555 -536
- swift/container/auditor.py +14 -100
- swift/container/backend.py +552 -227
- swift/container/reconciler.py +126 -37
- swift/container/replicator.py +96 -22
- swift/container/server.py +397 -176
- swift/container/sharder.py +1580 -639
- swift/container/sync.py +94 -88
- swift/container/updater.py +53 -32
- swift/obj/auditor.py +153 -35
- swift/obj/diskfile.py +466 -217
- swift/obj/expirer.py +406 -124
- swift/obj/mem_diskfile.py +7 -4
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +523 -262
- swift/obj/replicator.py +249 -188
- swift/obj/server.py +213 -122
- swift/obj/ssync_receiver.py +145 -85
- swift/obj/ssync_sender.py +113 -54
- swift/obj/updater.py +653 -139
- swift/obj/watchers/__init__.py +0 -0
- swift/obj/watchers/dark_data.py +213 -0
- swift/proxy/controllers/account.py +11 -11
- swift/proxy/controllers/base.py +848 -604
- swift/proxy/controllers/container.py +452 -86
- swift/proxy/controllers/info.py +3 -2
- swift/proxy/controllers/obj.py +1009 -490
- swift/proxy/server.py +185 -112
- swift-2.35.0.dist-info/AUTHORS +501 -0
- swift-2.35.0.dist-info/LICENSE +202 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
- swift-2.35.0.dist-info/RECORD +201 -0
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
- swift-2.35.0.dist-info/pbr.json +1 -0
- swift/locale/de/LC_MESSAGES/swift.po +0 -1216
- swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
- swift/locale/es/LC_MESSAGES/swift.po +0 -1085
- swift/locale/fr/LC_MESSAGES/swift.po +0 -909
- swift/locale/it/LC_MESSAGES/swift.po +0 -894
- swift/locale/ja/LC_MESSAGES/swift.po +0 -965
- swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
- swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
- swift/locale/ru/LC_MESSAGES/swift.po +0 -891
- swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
- swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
- swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
- swift-2.23.2.data/scripts/swift-account-auditor +0 -23
- swift-2.23.2.data/scripts/swift-account-info +0 -51
- swift-2.23.2.data/scripts/swift-account-reaper +0 -23
- swift-2.23.2.data/scripts/swift-account-replicator +0 -34
- swift-2.23.2.data/scripts/swift-account-server +0 -23
- swift-2.23.2.data/scripts/swift-container-auditor +0 -23
- swift-2.23.2.data/scripts/swift-container-info +0 -51
- swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
- swift-2.23.2.data/scripts/swift-container-replicator +0 -34
- swift-2.23.2.data/scripts/swift-container-sharder +0 -33
- swift-2.23.2.data/scripts/swift-container-sync +0 -23
- swift-2.23.2.data/scripts/swift-container-updater +0 -23
- swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
- swift-2.23.2.data/scripts/swift-form-signature +0 -20
- swift-2.23.2.data/scripts/swift-init +0 -119
- swift-2.23.2.data/scripts/swift-object-auditor +0 -29
- swift-2.23.2.data/scripts/swift-object-expirer +0 -33
- swift-2.23.2.data/scripts/swift-object-info +0 -60
- swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
- swift-2.23.2.data/scripts/swift-object-relinker +0 -41
- swift-2.23.2.data/scripts/swift-object-replicator +0 -37
- swift-2.23.2.data/scripts/swift-object-server +0 -27
- swift-2.23.2.data/scripts/swift-object-updater +0 -23
- swift-2.23.2.data/scripts/swift-proxy-server +0 -23
- swift-2.23.2.data/scripts/swift-recon +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder +0 -24
- swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.23.2.data/scripts/swift-ring-composer +0 -22
- swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
- swift-2.23.2.dist-info/RECORD +0 -220
- swift-2.23.2.dist-info/metadata.json +0 -1
- swift-2.23.2.dist-info/pbr.json +0 -1
- {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -16,21 +16,25 @@
|
|
16
16
|
from base64 import standard_b64encode as b64encode
|
17
17
|
from base64 import standard_b64decode as b64decode
|
18
18
|
|
19
|
-
import
|
20
|
-
from six.moves.urllib.parse import quote
|
19
|
+
from urllib.parse import quote
|
21
20
|
|
22
21
|
from swift.common import swob
|
23
22
|
from swift.common.http import HTTP_OK
|
24
|
-
from swift.common.
|
23
|
+
from swift.common.middleware.versioned_writes.object_versioning import \
|
24
|
+
DELETE_MARKER_CONTENT_TYPE
|
25
|
+
from swift.common.utils import json, public, config_true_value, Timestamp, \
|
26
|
+
cap_length
|
27
|
+
from swift.common.registry import get_swift_info
|
25
28
|
|
26
29
|
from swift.common.middleware.s3api.controllers.base import Controller
|
27
|
-
from swift.common.middleware.s3api.etree import Element, SubElement,
|
28
|
-
fromstring, XMLSyntaxError, DocumentInvalid
|
29
|
-
from swift.common.middleware.s3api.s3response import
|
30
|
-
InvalidArgument, \
|
30
|
+
from swift.common.middleware.s3api.etree import Element, SubElement, \
|
31
|
+
tostring, fromstring, XMLSyntaxError, DocumentInvalid
|
32
|
+
from swift.common.middleware.s3api.s3response import \
|
33
|
+
HTTPOk, S3NotImplemented, InvalidArgument, \
|
31
34
|
MalformedXML, InvalidLocationConstraint, NoSuchBucket, \
|
32
|
-
BucketNotEmpty,
|
33
|
-
|
35
|
+
BucketNotEmpty, VersionedBucketNotEmpty, InternalError, \
|
36
|
+
ServiceUnavailable, NoSuchKey
|
37
|
+
from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX, S3Timestamp
|
34
38
|
|
35
39
|
MAX_PUT_BUCKET_BODY_SIZE = 10240
|
36
40
|
|
@@ -50,7 +54,10 @@ class BucketController(Controller):
|
|
50
54
|
try:
|
51
55
|
resp = req.get_response(self.app, 'HEAD')
|
52
56
|
if int(resp.sw_headers['X-Container-Object-Count']) > 0:
|
53
|
-
|
57
|
+
if resp.sw_headers.get('X-Container-Sysmeta-Versions-Enabled'):
|
58
|
+
raise VersionedBucketNotEmpty()
|
59
|
+
else:
|
60
|
+
raise BucketNotEmpty()
|
54
61
|
# FIXME: This extra HEAD saves unexpected segment deletion
|
55
62
|
# but if a complete multipart upload happen while cleanup
|
56
63
|
# segment container below, completed object may be missing its
|
@@ -94,170 +101,266 @@ class BucketController(Controller):
|
|
94
101
|
|
95
102
|
return HTTPOk(headers=resp.headers)
|
96
103
|
|
97
|
-
|
98
|
-
def GET(self, req):
|
99
|
-
"""
|
100
|
-
Handle GET Bucket (List Objects) request
|
101
|
-
"""
|
102
|
-
|
103
|
-
max_keys = req.get_validated_param(
|
104
|
-
'max-keys', self.conf.max_bucket_listing)
|
105
|
-
# TODO: Separate max_bucket_listing and default_bucket_listing
|
106
|
-
tag_max_keys = max_keys
|
107
|
-
max_keys = min(max_keys, self.conf.max_bucket_listing)
|
108
|
-
|
104
|
+
def _parse_request_options(self, req, max_keys):
|
109
105
|
encoding_type = req.params.get('encoding-type')
|
110
106
|
if encoding_type is not None and encoding_type != 'url':
|
111
107
|
err_msg = 'Invalid Encoding Method specified in Request'
|
112
108
|
raise InvalidArgument('encoding-type', encoding_type, err_msg)
|
113
109
|
|
110
|
+
# in order to judge that truncated is valid, check whether
|
111
|
+
# max_keys + 1 th element exists in swift.
|
114
112
|
query = {
|
115
|
-
'format': 'json',
|
116
113
|
'limit': max_keys + 1,
|
117
114
|
}
|
118
115
|
if 'prefix' in req.params:
|
119
|
-
query
|
116
|
+
query['prefix'] = swob.wsgi_to_str(req.params['prefix'])
|
120
117
|
if 'delimiter' in req.params:
|
121
|
-
query
|
118
|
+
query['delimiter'] = swob.wsgi_to_str(req.params['delimiter'])
|
122
119
|
fetch_owner = False
|
123
120
|
if 'versions' in req.params:
|
121
|
+
query['versions'] = swob.wsgi_to_str(req.params['versions'])
|
124
122
|
listing_type = 'object-versions'
|
123
|
+
version_marker = swob.wsgi_to_str(req.params.get(
|
124
|
+
'version-id-marker'))
|
125
125
|
if 'key-marker' in req.params:
|
126
|
-
query
|
127
|
-
|
126
|
+
query['marker'] = swob.wsgi_to_str(req.params['key-marker'])
|
127
|
+
if version_marker is not None:
|
128
|
+
if version_marker != 'null':
|
129
|
+
try:
|
130
|
+
Timestamp(version_marker)
|
131
|
+
except ValueError:
|
132
|
+
raise InvalidArgument(
|
133
|
+
'version-id-marker', version_marker,
|
134
|
+
'Invalid version id specified')
|
135
|
+
query['version_marker'] = version_marker
|
136
|
+
elif version_marker is not None:
|
128
137
|
err_msg = ('A version-id marker cannot be specified without '
|
129
138
|
'a key marker.')
|
130
139
|
raise InvalidArgument('version-id-marker',
|
131
|
-
|
140
|
+
version_marker, err_msg)
|
132
141
|
elif int(req.params.get('list-type', '1')) == 2:
|
133
142
|
listing_type = 'version-2'
|
134
143
|
if 'start-after' in req.params:
|
135
|
-
query
|
144
|
+
query['marker'] = swob.wsgi_to_str(req.params['start-after'])
|
136
145
|
# continuation-token overrides start-after
|
137
146
|
if 'continuation-token' in req.params:
|
138
|
-
decoded = b64decode(
|
139
|
-
|
140
|
-
|
141
|
-
query.update({'marker': decoded})
|
147
|
+
decoded = b64decode(
|
148
|
+
req.params['continuation-token']).decode('utf8')
|
149
|
+
query['marker'] = decoded
|
142
150
|
if 'fetch-owner' in req.params:
|
143
151
|
fetch_owner = config_true_value(req.params['fetch-owner'])
|
144
152
|
else:
|
145
153
|
listing_type = 'version-1'
|
146
154
|
if 'marker' in req.params:
|
147
|
-
query
|
148
|
-
|
149
|
-
|
155
|
+
query['marker'] = swob.wsgi_to_str(req.params['marker'])
|
156
|
+
|
157
|
+
return encoding_type, query, listing_type, fetch_owner
|
158
|
+
|
159
|
+
def _build_versions_result(self, req, objects, encoding_type,
|
160
|
+
tag_max_keys, is_truncated):
|
161
|
+
elem = Element('ListVersionsResult')
|
162
|
+
SubElement(elem, 'Name').text = req.container_name
|
163
|
+
prefix = swob.wsgi_to_str(req.params.get('prefix'))
|
164
|
+
if prefix and encoding_type == 'url':
|
165
|
+
prefix = quote(prefix)
|
166
|
+
SubElement(elem, 'Prefix').text = prefix
|
167
|
+
key_marker = swob.wsgi_to_str(req.params.get('key-marker'))
|
168
|
+
if key_marker and encoding_type == 'url':
|
169
|
+
key_marker = quote(key_marker)
|
170
|
+
SubElement(elem, 'KeyMarker').text = key_marker
|
171
|
+
SubElement(elem, 'VersionIdMarker').text = swob.wsgi_to_str(
|
172
|
+
req.params.get('version-id-marker'))
|
173
|
+
if is_truncated:
|
174
|
+
if 'name' in objects[-1]:
|
175
|
+
SubElement(elem, 'NextKeyMarker').text = \
|
176
|
+
objects[-1]['name']
|
177
|
+
SubElement(elem, 'NextVersionIdMarker').text = \
|
178
|
+
objects[-1].get('version') or 'null'
|
179
|
+
if 'subdir' in objects[-1]:
|
180
|
+
SubElement(elem, 'NextKeyMarker').text = \
|
181
|
+
objects[-1]['subdir']
|
182
|
+
SubElement(elem, 'NextVersionIdMarker').text = 'null'
|
183
|
+
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
|
184
|
+
delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
|
185
|
+
if delimiter is not None:
|
186
|
+
if encoding_type == 'url':
|
187
|
+
delimiter = quote(delimiter)
|
188
|
+
SubElement(elem, 'Delimiter').text = delimiter
|
189
|
+
if encoding_type == 'url':
|
190
|
+
SubElement(elem, 'EncodingType').text = encoding_type
|
191
|
+
SubElement(elem, 'IsTruncated').text = \
|
192
|
+
'true' if is_truncated else 'false'
|
193
|
+
return elem
|
194
|
+
|
195
|
+
def _build_base_listing_element(self, req, encoding_type):
|
196
|
+
elem = Element('ListBucketResult')
|
197
|
+
SubElement(elem, 'Name').text = req.container_name
|
198
|
+
prefix = swob.wsgi_to_str(req.params.get('prefix'))
|
199
|
+
if prefix and encoding_type == 'url':
|
200
|
+
prefix = quote(prefix)
|
201
|
+
SubElement(elem, 'Prefix').text = prefix
|
202
|
+
return elem
|
203
|
+
|
204
|
+
def _build_list_bucket_result_type_one(self, req, objects, encoding_type,
|
205
|
+
tag_max_keys, is_truncated):
|
206
|
+
elem = self._build_base_listing_element(req, encoding_type)
|
207
|
+
marker = swob.wsgi_to_str(req.params.get('marker'))
|
208
|
+
if marker and encoding_type == 'url':
|
209
|
+
marker = quote(marker)
|
210
|
+
SubElement(elem, 'Marker').text = marker
|
211
|
+
if is_truncated and 'delimiter' in req.params:
|
212
|
+
if 'name' in objects[-1]:
|
213
|
+
name = objects[-1]['name']
|
214
|
+
else:
|
215
|
+
name = objects[-1]['subdir']
|
216
|
+
if encoding_type == 'url':
|
217
|
+
name = quote(name.encode('utf-8'))
|
218
|
+
SubElement(elem, 'NextMarker').text = name
|
219
|
+
# XXX: really? no NextMarker when no delimiter??
|
220
|
+
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
|
221
|
+
delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
|
222
|
+
if delimiter:
|
223
|
+
if encoding_type == 'url':
|
224
|
+
delimiter = quote(delimiter)
|
225
|
+
SubElement(elem, 'Delimiter').text = delimiter
|
226
|
+
if encoding_type == 'url':
|
227
|
+
SubElement(elem, 'EncodingType').text = encoding_type
|
228
|
+
SubElement(elem, 'IsTruncated').text = \
|
229
|
+
'true' if is_truncated else 'false'
|
230
|
+
return elem
|
231
|
+
|
232
|
+
def _build_list_bucket_result_type_two(self, req, objects, encoding_type,
|
233
|
+
tag_max_keys, is_truncated):
|
234
|
+
elem = self._build_base_listing_element(req, encoding_type)
|
235
|
+
if is_truncated:
|
236
|
+
if 'name' in objects[-1]:
|
237
|
+
SubElement(elem, 'NextContinuationToken').text = \
|
238
|
+
b64encode(objects[-1]['name'].encode('utf8'))
|
239
|
+
if 'subdir' in objects[-1]:
|
240
|
+
SubElement(elem, 'NextContinuationToken').text = \
|
241
|
+
b64encode(objects[-1]['subdir'].encode('utf8'))
|
242
|
+
if 'continuation-token' in req.params:
|
243
|
+
SubElement(elem, 'ContinuationToken').text = \
|
244
|
+
swob.wsgi_to_str(req.params['continuation-token'])
|
245
|
+
start_after = swob.wsgi_to_str(req.params.get('start-after'))
|
246
|
+
if start_after is not None:
|
247
|
+
if encoding_type == 'url':
|
248
|
+
start_after = quote(start_after)
|
249
|
+
SubElement(elem, 'StartAfter').text = start_after
|
250
|
+
SubElement(elem, 'KeyCount').text = str(len(objects))
|
251
|
+
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
|
252
|
+
delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
|
253
|
+
if delimiter:
|
254
|
+
if encoding_type == 'url':
|
255
|
+
delimiter = quote(delimiter)
|
256
|
+
SubElement(elem, 'Delimiter').text = delimiter
|
257
|
+
if encoding_type == 'url':
|
258
|
+
SubElement(elem, 'EncodingType').text = encoding_type
|
259
|
+
SubElement(elem, 'IsTruncated').text = \
|
260
|
+
'true' if is_truncated else 'false'
|
261
|
+
return elem
|
150
262
|
|
151
|
-
|
263
|
+
def _add_subdir(self, elem, o, encoding_type):
|
264
|
+
common_prefixes = SubElement(elem, 'CommonPrefixes')
|
265
|
+
name = o['subdir']
|
266
|
+
if encoding_type == 'url':
|
267
|
+
name = quote(name.encode('utf-8'))
|
268
|
+
SubElement(common_prefixes, 'Prefix').text = name
|
152
269
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
270
|
+
def _add_object(self, req, elem, o, encoding_type, listing_type,
|
271
|
+
fetch_owner):
|
272
|
+
name = o['name']
|
273
|
+
if encoding_type == 'url':
|
274
|
+
name = quote(name.encode('utf-8'))
|
157
275
|
|
158
276
|
if listing_type == 'object-versions':
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
SubElement(
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
objects[-1]['subdir']
|
172
|
-
SubElement(elem, 'NextVersionIdMarker').text = 'null'
|
277
|
+
if o['content_type'] == DELETE_MARKER_CONTENT_TYPE:
|
278
|
+
contents = SubElement(elem, 'DeleteMarker')
|
279
|
+
else:
|
280
|
+
contents = SubElement(elem, 'Version')
|
281
|
+
SubElement(contents, 'Key').text = name
|
282
|
+
SubElement(contents, 'VersionId').text = o.get(
|
283
|
+
'version_id') or 'null'
|
284
|
+
if 'object_versioning' in get_swift_info():
|
285
|
+
SubElement(contents, 'IsLatest').text = (
|
286
|
+
'true' if o['is_latest'] else 'false')
|
287
|
+
else:
|
288
|
+
SubElement(contents, 'IsLatest').text = 'true'
|
173
289
|
else:
|
174
|
-
|
175
|
-
SubElement(
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
if
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
290
|
+
contents = SubElement(elem, 'Contents')
|
291
|
+
SubElement(contents, 'Key').text = name
|
292
|
+
SubElement(contents, 'LastModified').text = \
|
293
|
+
S3Timestamp.from_isoformat(o['last_modified']).s3xmlformat
|
294
|
+
if contents.tag != 'DeleteMarker':
|
295
|
+
if 's3_etag' in o:
|
296
|
+
# New-enough MUs are already in the right format
|
297
|
+
etag = o['s3_etag']
|
298
|
+
elif 'slo_etag' in o:
|
299
|
+
# SLOs may be in something *close* to the MU format
|
300
|
+
etag = '"%s-N"' % o['slo_etag'].strip('"')
|
301
|
+
else:
|
302
|
+
# Normal objects just use the MD5
|
303
|
+
etag = o['hash']
|
304
|
+
if len(etag) < 2 or etag[::len(etag) - 1] != '""':
|
305
|
+
# Normal objects just use the MD5
|
306
|
+
etag = '"%s"' % o['hash']
|
307
|
+
# This also catches sufficiently-old SLOs, but we have
|
308
|
+
# no way to identify those from container listings
|
309
|
+
# Otherwise, somebody somewhere (proxyfs, maybe?) made this
|
310
|
+
# look like an RFC-compliant ETag; we don't need to
|
311
|
+
# quote-wrap.
|
312
|
+
SubElement(contents, 'ETag').text = etag
|
313
|
+
SubElement(contents, 'Size').text = str(o['bytes'])
|
314
|
+
if fetch_owner or listing_type != 'version-2':
|
315
|
+
owner = SubElement(contents, 'Owner')
|
316
|
+
SubElement(owner, 'ID').text = req.user_id
|
317
|
+
SubElement(owner, 'DisplayName').text = req.user_id
|
318
|
+
if contents.tag != 'DeleteMarker':
|
319
|
+
SubElement(contents, 'StorageClass').text = 'STANDARD'
|
320
|
+
|
321
|
+
def _add_objects_to_result(self, req, elem, objects, encoding_type,
|
322
|
+
listing_type, fetch_owner):
|
323
|
+
for o in objects:
|
324
|
+
if 'subdir' in o:
|
325
|
+
self._add_subdir(elem, o, encoding_type)
|
326
|
+
else:
|
327
|
+
self._add_object(req, elem, o, encoding_type, listing_type,
|
328
|
+
fetch_owner)
|
202
329
|
|
203
|
-
|
330
|
+
@public
|
331
|
+
def GET(self, req):
|
332
|
+
"""
|
333
|
+
Handle GET Bucket (List Objects) request
|
334
|
+
"""
|
335
|
+
tag_max_keys = req.get_validated_param(
|
336
|
+
'max-keys', self.conf.max_bucket_listing)
|
337
|
+
# TODO: Separate max_bucket_listing and default_bucket_listing
|
338
|
+
max_keys = min(tag_max_keys, self.conf.max_bucket_listing)
|
204
339
|
|
205
|
-
|
206
|
-
|
340
|
+
encoding_type, query, listing_type, fetch_owner = \
|
341
|
+
self._parse_request_options(req, max_keys)
|
207
342
|
|
208
|
-
|
209
|
-
SubElement(elem, 'EncodingType').text = encoding_type
|
343
|
+
resp = req.get_response(self.app, query=query)
|
210
344
|
|
211
|
-
|
212
|
-
|
345
|
+
try:
|
346
|
+
objects = json.loads(resp.body)
|
347
|
+
except (TypeError, ValueError):
|
348
|
+
self.logger.error('Got non-JSON response trying to list %s: %r',
|
349
|
+
req.path, cap_length(resp.body, 60))
|
350
|
+
raise
|
213
351
|
|
214
|
-
|
215
|
-
|
216
|
-
name = o['name']
|
217
|
-
if encoding_type == 'url':
|
218
|
-
name = quote(name.encode('utf-8'))
|
219
|
-
|
220
|
-
if listing_type == 'object-versions':
|
221
|
-
contents = SubElement(elem, 'Version')
|
222
|
-
SubElement(contents, 'Key').text = name
|
223
|
-
SubElement(contents, 'VersionId').text = 'null'
|
224
|
-
SubElement(contents, 'IsLatest').text = 'true'
|
225
|
-
else:
|
226
|
-
contents = SubElement(elem, 'Contents')
|
227
|
-
SubElement(contents, 'Key').text = name
|
228
|
-
SubElement(contents, 'LastModified').text = \
|
229
|
-
o['last_modified'][:-3] + 'Z'
|
230
|
-
if 's3_etag' in o:
|
231
|
-
# New-enough MUs are already in the right format
|
232
|
-
etag = o['s3_etag']
|
233
|
-
elif 'slo_etag' in o:
|
234
|
-
# SLOs may be in something *close* to the MU format
|
235
|
-
etag = '"%s-N"' % o['slo_etag'].strip('"')
|
236
|
-
else:
|
237
|
-
etag = o['hash']
|
238
|
-
if len(etag) < 2 or etag[::len(etag) - 1] != '""':
|
239
|
-
# Normal objects just use the MD5
|
240
|
-
etag = '"%s"' % o['hash']
|
241
|
-
# This also catches sufficiently-old SLOs, but we have
|
242
|
-
# no way to identify those from container listings
|
243
|
-
# Otherwise, somebody somewhere (proxyfs, maybe?) made this
|
244
|
-
# look like an RFC-compliant ETag; we don't need to
|
245
|
-
# quote-wrap.
|
246
|
-
SubElement(contents, 'ETag').text = etag
|
247
|
-
SubElement(contents, 'Size').text = str(o['bytes'])
|
248
|
-
if fetch_owner or listing_type != 'version-2':
|
249
|
-
owner = SubElement(contents, 'Owner')
|
250
|
-
SubElement(owner, 'ID').text = req.user_id
|
251
|
-
SubElement(owner, 'DisplayName').text = req.user_id
|
252
|
-
SubElement(contents, 'StorageClass').text = 'STANDARD'
|
352
|
+
is_truncated = max_keys > 0 and len(objects) > max_keys
|
353
|
+
objects = objects[:max_keys]
|
253
354
|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
355
|
+
if listing_type == 'object-versions':
|
356
|
+
func = self._build_versions_result
|
357
|
+
elif listing_type == 'version-2':
|
358
|
+
func = self._build_list_bucket_result_type_two
|
359
|
+
else:
|
360
|
+
func = self._build_list_bucket_result_type_one
|
361
|
+
elem = func(req, objects, encoding_type, tag_max_keys, is_truncated)
|
362
|
+
self._add_objects_to_result(
|
363
|
+
req, elem, objects, encoding_type, listing_type, fetch_owner)
|
261
364
|
|
262
365
|
body = tostring(elem)
|
263
366
|
|
@@ -281,7 +384,8 @@ class BucketController(Controller):
|
|
281
384
|
self.logger.error(e)
|
282
385
|
raise
|
283
386
|
|
284
|
-
if location
|
387
|
+
if location not in (self.conf.location,
|
388
|
+
self.conf.location.lower()):
|
285
389
|
# s3api cannot support multiple regions currently.
|
286
390
|
raise InvalidLocationConstraint()
|
287
391
|
|
@@ -297,6 +401,7 @@ class BucketController(Controller):
|
|
297
401
|
"""
|
298
402
|
Handle DELETE Bucket request
|
299
403
|
"""
|
404
|
+
# NB: object_versioning is responsible for cleaning up its container
|
300
405
|
if self.conf.allow_multipart_uploads:
|
301
406
|
self._delete_segments_bucket(req)
|
302
407
|
resp = req.get_response(self.app)
|
@@ -18,8 +18,8 @@ from swift.common.utils import public
|
|
18
18
|
from swift.common.middleware.s3api.controllers.base import Controller, \
|
19
19
|
bucket_operation
|
20
20
|
from swift.common.middleware.s3api.etree import Element, tostring
|
21
|
-
from swift.common.middleware.s3api.s3response import
|
22
|
-
NoLoggingStatusForKey
|
21
|
+
from swift.common.middleware.s3api.s3response import (
|
22
|
+
HTTPOk, S3NotImplemented, NoLoggingStatusForKey)
|
23
23
|
|
24
24
|
|
25
25
|
class LoggingStatusController(Controller):
|
@@ -17,15 +17,18 @@ import copy
|
|
17
17
|
import json
|
18
18
|
|
19
19
|
from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
|
20
|
+
from swift.common.http import HTTP_NO_CONTENT
|
21
|
+
from swift.common.swob import str_to_wsgi
|
20
22
|
from swift.common.utils import public, StreamingPile
|
23
|
+
from swift.common.registry import get_swift_info
|
21
24
|
|
22
25
|
from swift.common.middleware.s3api.controllers.base import Controller, \
|
23
26
|
bucket_operation
|
24
27
|
from swift.common.middleware.s3api.etree import Element, SubElement, \
|
25
28
|
fromstring, tostring, XMLSyntaxError, DocumentInvalid
|
26
|
-
from swift.common.middleware.s3api.s3response import HTTPOk,
|
27
|
-
NoSuchKey, ErrorResponse, MalformedXML,
|
28
|
-
AccessDenied, MissingRequestBodyError
|
29
|
+
from swift.common.middleware.s3api.s3response import HTTPOk, \
|
30
|
+
S3NotImplemented, NoSuchKey, ErrorResponse, MalformedXML, \
|
31
|
+
UserKeyMustBeSpecified, AccessDenied, MissingRequestBodyError
|
29
32
|
|
30
33
|
|
31
34
|
class MultiObjectDeleteController(Controller):
|
@@ -35,12 +38,10 @@ class MultiObjectDeleteController(Controller):
|
|
35
38
|
"""
|
36
39
|
def _gen_error_body(self, error, elem, delete_list):
|
37
40
|
for key, version in delete_list:
|
38
|
-
if version is not None:
|
39
|
-
# TODO: delete the specific version of the object
|
40
|
-
raise S3NotImplemented()
|
41
|
-
|
42
41
|
error_elem = SubElement(elem, 'Error')
|
43
42
|
SubElement(error_elem, 'Key').text = key
|
43
|
+
if version is not None:
|
44
|
+
SubElement(error_elem, 'VersionId').text = version
|
44
45
|
SubElement(error_elem, 'Code').text = error.__class__.__name__
|
45
46
|
SubElement(error_elem, 'Message').text = error._msg
|
46
47
|
|
@@ -76,14 +77,16 @@ class MultiObjectDeleteController(Controller):
|
|
76
77
|
if not xml:
|
77
78
|
raise MissingRequestBodyError()
|
78
79
|
|
79
|
-
req.
|
80
|
+
if 'x-amz-content-sha256' not in req.headers:
|
81
|
+
# SHA256 got checked when we read the body, so there's at
|
82
|
+
# least *some* verification. Recent versions of boto3 stopped
|
83
|
+
# sending Content-MD5, so it can't *always* be required.
|
84
|
+
# See https://bugs.launchpad.net/swift/+bug/2098529
|
85
|
+
req.check_md5(xml)
|
80
86
|
elem = fromstring(xml, 'Delete', self.logger)
|
81
87
|
|
82
88
|
quiet = elem.find('./Quiet')
|
83
|
-
|
84
|
-
self.quiet = True
|
85
|
-
else:
|
86
|
-
self.quiet = False
|
89
|
+
self.quiet = quiet is not None and quiet.text.lower() == 'true'
|
87
90
|
|
88
91
|
delete_list = list(object_key_iter(elem))
|
89
92
|
if len(delete_list) > self.conf.max_multi_delete_objects:
|
@@ -105,21 +108,35 @@ class MultiObjectDeleteController(Controller):
|
|
105
108
|
body = self._gen_error_body(error, elem, delete_list)
|
106
109
|
return HTTPOk(body=body)
|
107
110
|
|
108
|
-
if
|
109
|
-
|
111
|
+
if 'object_versioning' not in get_swift_info() and any(
|
112
|
+
version not in ('null', None)
|
113
|
+
for _key, version in delete_list):
|
110
114
|
raise S3NotImplemented()
|
111
115
|
|
112
116
|
def do_delete(base_req, key, version):
|
113
117
|
req = copy.copy(base_req)
|
114
118
|
req.environ = copy.copy(base_req.environ)
|
115
|
-
req.object_name = key
|
119
|
+
req.object_name = str_to_wsgi(key)
|
120
|
+
if version:
|
121
|
+
req.params = {'version-id': version, 'symlink': 'get'}
|
116
122
|
|
117
123
|
try:
|
118
|
-
|
124
|
+
try:
|
125
|
+
query = req.gen_multipart_manifest_delete_query(
|
126
|
+
self.app, version=version)
|
127
|
+
except NoSuchKey:
|
128
|
+
query = {}
|
129
|
+
if version:
|
130
|
+
query['version-id'] = version
|
131
|
+
query['symlink'] = 'get'
|
132
|
+
|
119
133
|
resp = req.get_response(self.app, method='DELETE', query=query,
|
120
134
|
headers={'Accept': 'application/json'})
|
121
|
-
#
|
122
|
-
|
135
|
+
# If async segment cleanup is available, we expect to get
|
136
|
+
# back a 204; otherwise, the delete is synchronous and we
|
137
|
+
# have to read the response to actually do the SLO delete
|
138
|
+
if query.get('multipart-manifest') and \
|
139
|
+
resp.status_int != HTTP_NO_CONTENT:
|
123
140
|
try:
|
124
141
|
delete_result = json.loads(resp.body)
|
125
142
|
if delete_result['Errors']:
|
@@ -135,8 +152,8 @@ class MultiObjectDeleteController(Controller):
|
|
135
152
|
except (ValueError, TypeError, KeyError):
|
136
153
|
# Logs get all the gory details
|
137
154
|
self.logger.exception(
|
138
|
-
'Could not parse SLO delete response: %
|
139
|
-
resp.body)
|
155
|
+
'Could not parse SLO delete response (%s): %s',
|
156
|
+
resp.status, resp.body)
|
140
157
|
# Client gets something more generic
|
141
158
|
return key, {'code': 'SLODeleteError',
|
142
159
|
'message': 'Unexpected swift response'}
|
@@ -144,6 +161,12 @@ class MultiObjectDeleteController(Controller):
|
|
144
161
|
pass
|
145
162
|
except ErrorResponse as e:
|
146
163
|
return key, {'code': e.__class__.__name__, 'message': e._msg}
|
164
|
+
except Exception:
|
165
|
+
self.logger.exception(
|
166
|
+
'Unexpected Error handling DELETE of %r %r' % (
|
167
|
+
req.container_name, key))
|
168
|
+
return key, {'code': 'Server Error', 'message': 'Server Error'}
|
169
|
+
|
147
170
|
return key, None
|
148
171
|
|
149
172
|
with StreamingPile(self.conf.multi_delete_concurrency) as pile:
|