PyPI - swift - Versions diffs - 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl - Mend

swift 2.23.3py3-none-any.whl → 2.35.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (206) hide show

swift/__init__.py +29 -50
swift/account/auditor.py +21 -118
swift/account/backend.py +33 -28
swift/account/reaper.py +37 -28
swift/account/replicator.py +22 -0
swift/account/server.py +60 -26
swift/account/utils.py +28 -11
swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
swift-2.23.3.data/scripts/swift-config → swift/cli/config.py +2 -2
swift/cli/container_deleter.py +5 -11
swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
swift/cli/dispersion_report.py +10 -9
swift-2.23.3.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
swift/cli/form_signature.py +3 -7
swift-2.23.3.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
swift/cli/info.py +154 -14
swift/cli/manage_shard_ranges.py +705 -37
swift-2.23.3.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
swift-2.23.3.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
swift/cli/recon.py +196 -67
swift-2.23.3.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
swift-2.23.3.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
swift/cli/relinker.py +807 -126
swift/cli/reload.py +135 -0
swift/cli/ringbuilder.py +217 -20
swift/cli/ringcomposer.py +0 -1
swift/cli/shard-info.py +4 -3
swift/common/base_storage_server.py +9 -20
swift/common/bufferedhttp.py +48 -74
swift/common/constraints.py +20 -15
swift/common/container_sync_realms.py +9 -11
swift/common/daemon.py +25 -8
swift/common/db.py +195 -128
swift/common/db_auditor.py +168 -0
swift/common/db_replicator.py +95 -55
swift/common/digest.py +141 -0
swift/common/direct_client.py +144 -33
swift/common/error_limiter.py +93 -0
swift/common/exceptions.py +25 -1
swift/common/header_key_dict.py +2 -9
swift/common/http_protocol.py +373 -0
swift/common/internal_client.py +129 -59
swift/common/linkat.py +3 -4
swift/common/manager.py +284 -67
swift/common/memcached.py +390 -145
swift/common/middleware/__init__.py +4 -0
swift/common/middleware/account_quotas.py +211 -46
swift/common/middleware/acl.py +3 -8
swift/common/middleware/backend_ratelimit.py +230 -0
swift/common/middleware/bulk.py +22 -34
swift/common/middleware/catch_errors.py +1 -3
swift/common/middleware/cname_lookup.py +6 -11
swift/common/middleware/container_quotas.py +1 -1
swift/common/middleware/container_sync.py +39 -17
swift/common/middleware/copy.py +12 -0
swift/common/middleware/crossdomain.py +22 -9
swift/common/middleware/crypto/__init__.py +2 -1
swift/common/middleware/crypto/crypto_utils.py +11 -15
swift/common/middleware/crypto/decrypter.py +28 -11
swift/common/middleware/crypto/encrypter.py +12 -17
swift/common/middleware/crypto/keymaster.py +8 -15
swift/common/middleware/crypto/kms_keymaster.py +2 -1
swift/common/middleware/dlo.py +15 -11
swift/common/middleware/domain_remap.py +5 -4
swift/common/middleware/etag_quoter.py +128 -0
swift/common/middleware/formpost.py +73 -70
swift/common/middleware/gatekeeper.py +8 -1
swift/common/middleware/keystoneauth.py +33 -3
swift/common/middleware/list_endpoints.py +4 -4
swift/common/middleware/listing_formats.py +85 -49
swift/common/middleware/memcache.py +4 -95
swift/common/middleware/name_check.py +3 -2
swift/common/middleware/proxy_logging.py +160 -92
swift/common/middleware/ratelimit.py +17 -10
swift/common/middleware/read_only.py +6 -4
swift/common/middleware/recon.py +59 -22
swift/common/middleware/s3api/acl_handlers.py +25 -3
swift/common/middleware/s3api/acl_utils.py +6 -1
swift/common/middleware/s3api/controllers/__init__.py +6 -0
swift/common/middleware/s3api/controllers/acl.py +3 -2
swift/common/middleware/s3api/controllers/bucket.py +242 -137
swift/common/middleware/s3api/controllers/logging.py +2 -2
swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
swift/common/middleware/s3api/controllers/obj.py +112 -8
swift/common/middleware/s3api/controllers/object_lock.py +44 -0
swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
swift/common/middleware/s3api/controllers/tagging.py +57 -0
swift/common/middleware/s3api/controllers/versioning.py +36 -7
swift/common/middleware/s3api/etree.py +22 -9
swift/common/middleware/s3api/exception.py +0 -4
swift/common/middleware/s3api/s3api.py +113 -41
swift/common/middleware/s3api/s3request.py +384 -218
swift/common/middleware/s3api/s3response.py +126 -23
swift/common/middleware/s3api/s3token.py +16 -17
swift/common/middleware/s3api/schema/delete.rng +1 -1
swift/common/middleware/s3api/subresource.py +7 -10
swift/common/middleware/s3api/utils.py +27 -10
swift/common/middleware/slo.py +665 -358
swift/common/middleware/staticweb.py +64 -37
swift/common/middleware/symlink.py +51 -18
swift/common/middleware/tempauth.py +76 -58
swift/common/middleware/tempurl.py +191 -173
swift/common/middleware/versioned_writes/__init__.py +51 -0
swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
swift/common/middleware/x_profile/exceptions.py +1 -4
swift/common/middleware/x_profile/html_viewer.py +18 -19
swift/common/middleware/x_profile/profile_model.py +1 -2
swift/common/middleware/xprofile.py +10 -10
swift-2.23.3.data/scripts/swift-container-server → swift/common/recon.py +13 -8
swift/common/registry.py +147 -0
swift/common/request_helpers.py +324 -57
swift/common/ring/builder.py +67 -25
swift/common/ring/composite_builder.py +1 -1
swift/common/ring/ring.py +177 -51
swift/common/ring/utils.py +1 -1
swift/common/splice.py +10 -6
swift/common/statsd_client.py +205 -0
swift/common/storage_policy.py +49 -44
swift/common/swob.py +86 -102
swift/common/{utils.py → utils/__init__.py} +2163 -2772
swift/common/utils/base.py +131 -0
swift/common/utils/config.py +433 -0
swift/common/utils/ipaddrs.py +256 -0
swift/common/utils/libc.py +345 -0
swift/common/utils/logs.py +859 -0
swift/common/utils/timestamp.py +412 -0
swift/common/wsgi.py +553 -535
swift/container/auditor.py +14 -100
swift/container/backend.py +490 -231
swift/container/reconciler.py +126 -37
swift/container/replicator.py +96 -22
swift/container/server.py +358 -165
swift/container/sharder.py +1540 -684
swift/container/sync.py +94 -88
swift/container/updater.py +53 -32
swift/obj/auditor.py +153 -35
swift/obj/diskfile.py +466 -217
swift/obj/expirer.py +406 -124
swift/obj/mem_diskfile.py +7 -4
swift/obj/mem_server.py +1 -0
swift/obj/reconstructor.py +523 -262
swift/obj/replicator.py +249 -188
swift/obj/server.py +207 -122
swift/obj/ssync_receiver.py +145 -85
swift/obj/ssync_sender.py +113 -54
swift/obj/updater.py +652 -139
swift/obj/watchers/__init__.py +0 -0
swift/obj/watchers/dark_data.py +213 -0
swift/proxy/controllers/account.py +11 -11
swift/proxy/controllers/base.py +848 -604
swift/proxy/controllers/container.py +433 -92
swift/proxy/controllers/info.py +3 -2
swift/proxy/controllers/obj.py +1000 -489
swift/proxy/server.py +185 -112
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/AUTHORS +58 -11
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/METADATA +51 -56
swift-2.35.0.dist-info/RECORD +201 -0
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
swift-2.35.0.dist-info/pbr.json +1 -0
swift/locale/de/LC_MESSAGES/swift.po +0 -1216
swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
swift/locale/es/LC_MESSAGES/swift.po +0 -1085
swift/locale/fr/LC_MESSAGES/swift.po +0 -909
swift/locale/it/LC_MESSAGES/swift.po +0 -894
swift/locale/ja/LC_MESSAGES/swift.po +0 -965
swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
swift/locale/ru/LC_MESSAGES/swift.po +0 -891
swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
swift-2.23.3.data/scripts/swift-account-auditor +0 -23
swift-2.23.3.data/scripts/swift-account-info +0 -51
swift-2.23.3.data/scripts/swift-account-reaper +0 -23
swift-2.23.3.data/scripts/swift-account-replicator +0 -34
swift-2.23.3.data/scripts/swift-account-server +0 -23
swift-2.23.3.data/scripts/swift-container-auditor +0 -23
swift-2.23.3.data/scripts/swift-container-info +0 -55
swift-2.23.3.data/scripts/swift-container-reconciler +0 -21
swift-2.23.3.data/scripts/swift-container-replicator +0 -34
swift-2.23.3.data/scripts/swift-container-sharder +0 -37
swift-2.23.3.data/scripts/swift-container-sync +0 -23
swift-2.23.3.data/scripts/swift-container-updater +0 -23
swift-2.23.3.data/scripts/swift-dispersion-report +0 -24
swift-2.23.3.data/scripts/swift-form-signature +0 -20
swift-2.23.3.data/scripts/swift-init +0 -119
swift-2.23.3.data/scripts/swift-object-auditor +0 -29
swift-2.23.3.data/scripts/swift-object-expirer +0 -33
swift-2.23.3.data/scripts/swift-object-info +0 -60
swift-2.23.3.data/scripts/swift-object-reconstructor +0 -33
swift-2.23.3.data/scripts/swift-object-relinker +0 -41
swift-2.23.3.data/scripts/swift-object-replicator +0 -37
swift-2.23.3.data/scripts/swift-object-server +0 -27
swift-2.23.3.data/scripts/swift-object-updater +0 -23
swift-2.23.3.data/scripts/swift-proxy-server +0 -23
swift-2.23.3.data/scripts/swift-recon +0 -24
swift-2.23.3.data/scripts/swift-ring-builder +0 -24
swift-2.23.3.data/scripts/swift-ring-builder-analyzer +0 -22
swift-2.23.3.data/scripts/swift-ring-composer +0 -22
swift-2.23.3.dist-info/RECORD +0 -220
swift-2.23.3.dist-info/pbr.json +0 -1
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/LICENSE +0 -0
{swift-2.23.3.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0

swift/common/middleware/staticweb.py CHANGED Viewed

@@ -59,7 +59,8 @@ requests for paths not found.
 For pseudo paths that have no <index.name>, this middleware can serve HTML file
 listings if you set the ``X-Container-Meta-Web-Listings: true`` metadata item
-on the container.
+on the container. Note that the listing must be authorized; you may want a
+container ACL like ``X-Container-Read: .r:*,.rlistings``.
 If listings are enabled, the listings can have a custom style sheet by setting
 the X-Container-Meta-Web-Listings-CSS header. For instance, setting
@@ -68,6 +69,17 @@ the .../listing.css style sheet. If you "view source" in your browser on a
 listing page, you will see the well defined document structure that can be
 styled.
+Additionally, prefix-based :ref:`tempurl` parameters may be used to authorize
+requests instead of making the whole container publicly readable. This gives
+clients dynamic discoverability of the objects available within that prefix.
+.. note::
+    ``temp_url_prefix`` values should typically end with a slash (``/``) when
+    used with StaticWeb. StaticWeb's redirects will not carry over any TempURL
+    parameters, as they likely indicate that the user created an overly-broad
+    TempURL.
 By default, the listings will be rendered with a label of
 "Listing of /v1/account/container/path".  This can be altered by
 setting a ``X-Container-Meta-Web-Listings-Label: <label>``.  For example,
@@ -123,19 +135,20 @@ Example usage of this middleware via ``swift``:
 """
-import cgi
+import html
 import json
-import six
 import time
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from swift.common.utils import human_readable, split_path, config_true_value, \
-    quote, register_swift_info, get_logger
+    quote, get_logger
+from swift.common.registry import register_swift_info
 from swift.common.wsgi import make_env, WSGIContext
 from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
 from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \
-    Request, wsgi_quote, wsgi_to_str
+    Request, wsgi_quote, wsgi_to_str, str_to_wsgi
+from swift.common.middleware.tempurl import get_temp_url_info
 from swift.proxy.controllers.base import get_container_info
@@ -224,13 +237,13 @@ class _StaticWebContext(WSGIContext):
             self._dir_type = meta.get('web-directory-type', '').strip()
         return container_info
-    def _listing(self, env, start_response, prefix=None):
+    def _listing(self, env, start_response, prefix=''):
         """
         Sends an HTML object listing to the remote client.
         :param env: The original WSGI environment dict.
         :param start_response: The original WSGI start_response hook.
-        :param prefix: Any prefix desired for the container listing.
+        :param prefix: Any WSGI-str prefix desired for the container listing.
         """
         label = wsgi_to_str(env['PATH_INFO'])
         if self._listings_label:
@@ -239,11 +252,10 @@ class _StaticWebContext(WSGIContext):
                                      '/'.join(groups[4:]))
         if not config_true_value(self._listings):
-            body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \
-                'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \
+            body = '<!DOCTYPE html>\n' \
                 '<html>\n' \
                 '<head>\n' \
-                '<title>Listing of %s</title>\n' % cgi.escape(label)
+                '<title>Listing of %s</title>\n' % html.escape(label)
             if self._listings_css:
                 body += '  <link rel="stylesheet" type="text/css" ' \
                     'href="%s" />\n' % self._build_css_path(prefix or '')
@@ -281,16 +293,35 @@ class _StaticWebContext(WSGIContext):
         body = b''.join(resp)
         if body:
             listing = json.loads(body)
-        if not listing:
+        if prefix and not listing:
             resp = HTTPNotFound()(env, self._start_response)
             return self._error_response(resp, env, start_response)
-        headers = {'Content-Type': 'text/html; charset=UTF-8'}
-        body = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 ' \
-               'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n' \
+        tempurl_qs = tempurl_prefix = ''
+        if env.get('REMOTE_USER') == '.wsgi.tempurl':
+            sig, expires, tempurl_prefix, _filename, inline, ip_range = \
+                get_temp_url_info(env)
+            if tempurl_prefix is None:
+                tempurl_prefix = ''
+            else:
+                parts = [
+                    'temp_url_prefix=%s' % quote(tempurl_prefix),
+                    'temp_url_expires=%s' % quote(str(expires)),
+                    'temp_url_sig=%s' % sig,
+                ]
+                if ip_range:
+                    parts.append('temp_url_ip_range=%s' % quote(ip_range))
+                if inline:
+                    parts.append('inline')
+                tempurl_qs = '?' + '&amp;'.join(parts)
+        headers = {'Content-Type': 'text/html; charset=UTF-8',
+                   'X-Backend-Content-Generator': 'staticweb'}
+        body = '<!DOCTYPE html>\n' \
                '<html>\n' \
                ' <head>\n' \
                '  <title>Listing of %s</title>\n' % \
-               cgi.escape(label)
+               html.escape(label)
         if self._listings_css:
             body += '  <link rel="stylesheet" type="text/css" ' \
                     'href="%s" />\n' % (self._build_css_path(prefix))
@@ -309,46 +340,42 @@ class _StaticWebContext(WSGIContext):
                 '    <th class="colname">Name</th>\n' \
                 '    <th class="colsize">Size</th>\n' \
                 '    <th class="coldate">Date</th>\n' \
-                '   </tr>\n' % cgi.escape(label)
-        if prefix:
+                '   </tr>\n' % html.escape(label)
+        if len(prefix) > len(tempurl_prefix):
             body += '   <tr id="parent" class="item">\n' \
-                    '    <td class="colname"><a href="../">../</a></td>\n' \
+                    '    <td class="colname"><a href="../%s">../</a></td>\n' \
                     '    <td class="colsize">&nbsp;</td>\n' \
                     '    <td class="coldate">&nbsp;</td>\n' \
-                    '   </tr>\n'
+                    '   </tr>\n' % tempurl_qs
         for item in listing:
             if 'subdir' in item:
-                subdir = item['subdir'] if six.PY3 else  \
-                    item['subdir'].encode('utf-8')
+                subdir = item['subdir']
                 if prefix:
-                    subdir = subdir[len(prefix):]
+                    subdir = subdir[len(wsgi_to_str(prefix)):]
                 body += '   <tr class="item subdir">\n' \
                         '    <td class="colname"><a href="%s">%s</a></td>\n' \
                         '    <td class="colsize">&nbsp;</td>\n' \
                         '    <td class="coldate">&nbsp;</td>\n' \
                         '   </tr>\n' % \
-                        (quote(subdir), cgi.escape(subdir))
+                        (quote(subdir) + tempurl_qs, html.escape(subdir))
         for item in listing:
             if 'name' in item:
-                name = item['name'] if six.PY3 else  \
-                    item['name'].encode('utf-8')
+                name = item['name']
                 if prefix:
-                    name = name[len(prefix):]
-                content_type = item['content_type'] if six.PY3 else  \
-                    item['content_type'].encode('utf-8')
+                    name = name[len(wsgi_to_str(prefix)):]
+                content_type = item['content_type']
                 bytes = human_readable(item['bytes'])
                 last_modified = (
-                    cgi.escape(item['last_modified'] if six.PY3 else
-                               item['last_modified'].encode('utf-8')).
+                    html.escape(item['last_modified']).
                     split('.')[0].replace('T', ' '))
                 body += '   <tr class="item %s">\n' \
                         '    <td class="colname"><a href="%s">%s</a></td>\n' \
                         '    <td class="colsize">%s</td>\n' \
                         '    <td class="coldate">%s</td>\n' \
                         '   </tr>\n' % \
-                        (' '.join('type-' + cgi.escape(t.lower(), quote=True)
+                        (' '.join('type-' + html.escape(t.lower())
                                   for t in content_type.split('/')),
-                         quote(name), cgi.escape(name),
+                         quote(name) + tempurl_qs, html.escape(name),
                          bytes, last_modified)
         body += '  </table>\n' \
                 ' </body>\n' \
@@ -408,7 +435,7 @@ class _StaticWebContext(WSGIContext):
         tmp_env['HTTP_USER_AGENT'] = \
             '%s StaticWeb' % env.get('HTTP_USER_AGENT')
         tmp_env['swift.source'] = 'SW'
-        tmp_env['PATH_INFO'] += self._index
+        tmp_env['PATH_INFO'] += str_to_wsgi(self._index)
         resp = self._app_call(tmp_env)
         status_int = self._get_status_int()
         if status_int == HTTP_NOT_FOUND:
@@ -465,7 +492,7 @@ class _StaticWebContext(WSGIContext):
             tmp_env['swift.source'] = 'SW'
             if not tmp_env['PATH_INFO'].endswith('/'):
                 tmp_env['PATH_INFO'] += '/'
-            tmp_env['PATH_INFO'] += self._index
+            tmp_env['PATH_INFO'] += str_to_wsgi(self._index)
             resp = self._app_call(tmp_env)
             status_int = self._get_status_int()
             if is_success(status_int) or is_redirection(status_int):
@@ -541,8 +568,8 @@ class StaticWeb(object):
             return self.app(env, start_response)
         if env['REQUEST_METHOD'] not in ('HEAD', 'GET'):
             return self.app(env, start_response)
-        if env.get('REMOTE_USER') and \
-                not config_true_value(env.get('HTTP_X_WEB_MODE', 'f')):
+        if env.get('REMOTE_USER') and env['REMOTE_USER'] != '.wsgi.tempurl' \
+                and not config_true_value(env.get('HTTP_X_WEB_MODE', 'f')):
             return self.app(env, start_response)
         if not container:
             return self.app(env, start_response)

swift/common/middleware/symlink.py CHANGED Viewed

@@ -199,17 +199,20 @@ configuration steps are required:
 import json
 import os
-from cgi import parse_header
-from swift.common.utils import get_logger, register_swift_info, split_path, \
-    MD5_OF_EMPTY_STRING, close_if_possible, closing_if_possible
+from swift.common.utils import get_logger, split_path, \
+    MD5_OF_EMPTY_STRING, close_if_possible, closing_if_possible, \
+    config_true_value, drain_and_close, parse_header
+from swift.common.registry import register_swift_info
 from swift.common.constraints import check_account_format
-from swift.common.wsgi import WSGIContext, make_subrequest
+from swift.common.wsgi import WSGIContext, make_subrequest, \
+    make_pre_authed_request
 from swift.common.request_helpers import get_sys_meta_prefix, \
-    check_path_header, get_container_update_override_key
+    check_path_header, get_container_update_override_key, \
+    update_ignore_range_header
 from swift.common.swob import Request, HTTPBadRequest, HTTPTemporaryRedirect, \
     HTTPException, HTTPConflict, HTTPPreconditionFailed, wsgi_quote, \
-    wsgi_unquote, status_map
+    wsgi_unquote, status_map, normalize_etag
 from swift.common.http import is_success, HTTP_NOT_FOUND
 from swift.common.exceptions import LinkIterError
 from swift.common.header_key_dict import HeaderKeyDict
@@ -227,6 +230,8 @@ TGT_ETAG_SYSMETA_SYMLINK_HDR = \
     get_sys_meta_prefix('object') + 'symlink-target-etag'
 TGT_BYTES_SYSMETA_SYMLINK_HDR = \
     get_sys_meta_prefix('object') + 'symlink-target-bytes'
+SYMLOOP_EXTEND = get_sys_meta_prefix('object') + 'symloop-extend'
+ALLOW_RESERVED_NAMES = get_sys_meta_prefix('object') + 'allow-reserved-names'
 def _validate_and_prep_request_headers(req):
@@ -281,9 +286,9 @@ def _validate_and_prep_request_headers(req):
         raise HTTPBadRequest(
             body='Symlink cannot target itself',
             request=req, content_type='text/plain')
-    etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None)
+    etag = normalize_etag(req.headers.get(TGT_ETAG_SYMLINK_HDR, None))
     if etag and any(c in etag for c in ';"\\'):
-        # See cgi.parse_header for why the above chars are problematic
+        # See utils.parse_header for why the above chars are problematic
         raise HTTPBadRequest(
             body='Bad %s format' % TGT_ETAG_SYMLINK_HDR.title(),
             request=req, content_type='text/plain')
@@ -428,6 +433,7 @@ class SymlinkObjectContext(WSGIContext):
         :param req: HTTP GET or HEAD object request
         :returns: Response Iterator
         """
+        update_ignore_range_header(req, TGT_OBJ_SYSMETA_SYMLINK_HDR)
         try:
             return self._recursive_get_head(req)
         except LinkIterError:
@@ -437,7 +443,9 @@ class SymlinkObjectContext(WSGIContext):
                                content_type='text/plain')
     def _recursive_get_head(self, req, target_etag=None,
-                            follow_softlinks=True):
+                            follow_softlinks=True, orig_req=None):
+        if not orig_req:
+            orig_req = req
         resp = self._app_call(req.environ)
         def build_traversal_req(symlink_target):
@@ -452,9 +460,20 @@ class SymlinkObjectContext(WSGIContext):
                 '/', version, account,
                 symlink_target.lstrip('/'))
             self._last_target_path = target_path
-            new_req = make_subrequest(
-                req.environ, path=target_path, method=req.method,
-                headers=req.headers, swift_source='SYM')
+            subreq_headers = dict(req.headers)
+            if self._response_header_value(ALLOW_RESERVED_NAMES):
+                # this symlink's sysmeta says it can point to reserved names,
+                # we're infering that some piece of middleware had previously
+                # authorized this request because users can't access reserved
+                # names directly
+                subreq_meth = make_pre_authed_request
+                subreq_headers['X-Backend-Allow-Reserved-Names'] = 'true'
+            else:
+                subreq_meth = make_subrequest
+            new_req = subreq_meth(orig_req.environ, path=target_path,
+                                  method=req.method, headers=subreq_headers,
+                                  swift_source='SYM')
             new_req.headers.pop('X-Backend-Storage-Policy-Index', None)
             return new_req
@@ -463,7 +482,8 @@ class SymlinkObjectContext(WSGIContext):
         resp_etag = self._response_header_value(
             TGT_ETAG_SYSMETA_SYMLINK_HDR)
         if symlink_target and (resp_etag or follow_softlinks):
-            close_if_possible(resp)
+            # Should be a zero-byte object
+            drain_and_close(resp)
             found_etag = resp_etag or self._response_header_value('etag')
             if target_etag and target_etag != found_etag:
                 raise HTTPConflict(
@@ -475,11 +495,15 @@ class SymlinkObjectContext(WSGIContext):
                 raise LinkIterError()
             # format: /<account name>/<container name>/<object name>
             new_req = build_traversal_req(symlink_target)
-            self._loop_count += 1
-            return self._recursive_get_head(new_req, target_etag=resp_etag)
+            if not config_true_value(
+                    self._response_header_value(SYMLOOP_EXTEND)):
+                self._loop_count += 1
+            return self._recursive_get_head(new_req, target_etag=resp_etag,
+                                            orig_req=req)
         else:
             final_etag = self._response_header_value('etag')
             if final_etag and target_etag and target_etag != final_etag:
+                # do *not* drain; we don't know how big this is
                 close_if_possible(resp)
                 body = ('Object Etag %r does not match '
                         'X-Symlink-Target-Etag header %r')
@@ -504,10 +528,18 @@ class SymlinkObjectContext(WSGIContext):
     def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
                                           etag):
+        if req.environ.get('swift.symlink_override'):
+            req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
+            req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
+                req.headers[TGT_BYTES_SYMLINK_HDR]
+            return
         # next we'll make sure the E-Tag matches a real object
         new_req = make_subrequest(
             req.environ, path=wsgi_quote(symlink_target_path), method='HEAD',
             swift_source='SYM')
+        if req.allow_reserved_names:
+            new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
         self._last_target_path = symlink_target_path
         resp = self._recursive_get_head(new_req, target_etag=etag,
                                         follow_softlinks=False)
@@ -519,9 +551,7 @@ class SymlinkObjectContext(WSGIContext):
                     'Content-Type': 'text/plain',
                     'Content-Location': self._last_target_path})
         if not is_success(self._get_status_int()):
-            with closing_if_possible(resp):
-                for chunk in resp:
-                    pass
+            drain_and_close(resp)
             raise status_map[self._get_status_int()](request=req)
         response_headers = HeaderKeyDict(self._response_headers)
         # carry forward any etag update params (e.g. "slo_etag"), we'll append
@@ -651,6 +681,9 @@ class SymlinkObjectContext(WSGIContext):
             req.environ['swift.leave_relative_location'] = True
             errmsg = 'The requested POST was applied to a symlink. POST ' +\
                      'directly to the target to apply requested metadata.'
+            for key, value in self._response_headers:
+                if key.lower().startswith('x-object-sysmeta-'):
+                    headers[key] = value
             raise HTTPTemporaryRedirect(
                 body=errmsg, headers=headers)
         else:

swift/common/middleware/tempauth.py CHANGED Viewed

@@ -54,12 +54,13 @@ in a line like this::
     user64_<account_b64>_<user_b64> = <key> [group] [...] [storage_url]
-There are two special groups:
+There are three special groups:
 * ``.reseller_admin`` -- can do anything to any account for this auth
+* ``.reseller_reader`` -- can GET/HEAD anything in any account for this auth
 * ``.admin`` -- can do anything within the account
-If neither of these groups are specified, the user can only access
+If none of these groups are specified, the user can only access
 containers that have been explicitly allowed for them by a ``.admin`` or
 ``.reseller_admin``.
@@ -82,16 +83,16 @@ Multiple Reseller Prefix Items
 The reseller prefix specifies which parts of the account namespace this
 middleware is responsible for managing authentication and authorization.
-By default, the prefix is 'AUTH' so accounts and tokens are prefixed
-by 'AUTH\_'. When a request's token and/or path start with 'AUTH\_', this
+By default, the prefix is ``AUTH`` so accounts and tokens are prefixed
+by ``AUTH_``. When a request's token and/or path start with ``AUTH_``, this
 middleware knows it is responsible.
 We allow the reseller prefix to be a list. In tempauth, the first item
 in the list is used as the prefix for tokens and user groups. The
 other prefixes provide alternate accounts that user's can access. For
-example if the reseller prefix list is 'AUTH, OTHER', a user with
-admin access to 'AUTH_account' also has admin access to
-'OTHER_account'.
+example if the reseller prefix list is ``AUTH, OTHER``, a user with
+admin access to ``AUTH_account`` also has admin access to
+``OTHER_account``.
 Required Group
 ^^^^^^^^^^^^^^
@@ -112,7 +113,7 @@ derived from the token are appended to the roles derived from
 The ``X-Service-Token`` is useful when combined with multiple reseller
 prefix items. In the following configuration, accounts prefixed
-``SERVICE\_`` are only accessible if ``X-Auth-Token`` is from the end-user
+``SERVICE_`` are only accessible if ``X-Auth-Token`` is from the end-user
 and ``X-Service-Token`` is from the ``glance`` user::
    [filter:tempauth]
@@ -124,8 +125,8 @@ and ``X-Service-Token`` is from the ``glance`` user::
    user_maryacct_mary = marypw .admin
    user_glance_glance = glancepw .service
-The name ``.service`` is an example. Unlike ``.admin`` and
-``.reseller_admin`` it is not a reserved name.
+The name ``.service`` is an example. Unlike ``.admin``, ``.reseller_admin``,
+``.reseller_reader`` it is not a reserved name.
 Please note that ACLs can be set on service accounts and are matched
 against the identity validated by ``X-Auth-Token``. As such ACLs can grant
@@ -172,7 +173,6 @@ To generate a curl command line from the above::
     '
 """
-from __future__ import print_function
 import json
 from time import time
@@ -181,16 +181,19 @@ from uuid import uuid4
 import base64
 from eventlet import Timeout
-import six
-from swift.common.swob import Response, Request, wsgi_to_str
-from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
-    HTTPUnauthorized, HTTPMethodNotAllowed
+from swift.common.memcached import MemcacheConnectionError
+from swift.common.swob import (
+    Response, Request, wsgi_to_str, str_to_wsgi, wsgi_unquote,
+    HTTPBadRequest, HTTPForbidden, HTTPNotFound,
+    HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable,
+)
 from swift.common.request_helpers import get_sys_meta_prefix
 from swift.common.middleware.acl import (
     clean_acl, parse_acl, referrer_allowed, acls_from_account_info)
 from swift.common.utils import cache_from_env, get_logger, \
-    split_path, config_true_value, register_swift_info
+    split_path, config_true_value
+from swift.common.registry import register_swift_info
 from swift.common.utils import config_read_reseller_options, quote
 from swift.proxy.controllers.base import get_account_info
@@ -207,13 +210,14 @@ class TempAuth(object):
     def __init__(self, app, conf):
         self.app = app
         self.conf = conf
-        self.logger = get_logger(conf, log_route='tempauth')
-        self.log_headers = config_true_value(conf.get('log_headers', 'f'))
         self.reseller_prefixes, self.account_rules = \
             config_read_reseller_options(conf, dict(require_group=''))
         self.reseller_prefix = self.reseller_prefixes[0]
-        self.logger.set_statsd_prefix('tempauth.%s' % (
-            self.reseller_prefix if self.reseller_prefix else 'NONE',))
+        statsd_tail_prefix = 'tempauth.%s' % (
+            self.reseller_prefix if self.reseller_prefix else 'NONE',)
+        self.logger = get_logger(conf, log_route='tempauth',
+                                 statsd_tail_prefix=statsd_tail_prefix)
+        self.log_headers = config_true_value(conf.get('log_headers', 'f'))
         self.auth_prefix = conf.get('auth_prefix', '/auth/')
         if not self.auth_prefix or not self.auth_prefix.strip('/'):
             self.logger.warning('Rewriting invalid auth prefix "%s" to '
@@ -231,17 +235,18 @@ class TempAuth(object):
         self.users = {}
         for conf_key in conf:
             if conf_key.startswith(('user_', 'user64_')):
-                account, username = conf_key.split('_', 1)[1].split('_')
+                try:
+                    account, username = conf_key.split('_', 1)[1].split('_')
+                except ValueError:
+                    raise ValueError("key %s was provided in an "
+                                     "invalid format" % conf_key)
                 if conf_key.startswith('user64_'):
                     # Because trailing equal signs would screw up config file
                     # parsing, we auto-pad with '=' chars.
                     account += '=' * (len(account) % 4)
-                    account = base64.b64decode(account)
+                    account = base64.b64decode(account).decode('utf8')
                     username += '=' * (len(username) % 4)
-                    username = base64.b64decode(username)
-                    if not six.PY2:
-                        account = account.decode('utf8')
-                        username = username.decode('utf8')
+                    username = base64.b64decode(username).decode('utf8')
                 values = conf[conf_key].split()
                 if not values:
                     raise ValueError('%s has no key set' % conf_key)
@@ -441,8 +446,6 @@ class TempAuth(object):
             expires, groups = cached_auth_data
             if expires < time():
                 groups = None
-            elif six.PY2:
-                groups = groups.encode('utf8')
         s3_auth_details = env.get('s3api.auth_details') or\
             env.get('swift3.auth_details')
@@ -461,7 +464,7 @@ class TempAuth(object):
             if not s3_auth_details['check_signature'](user['key']):
                 return None
             env['PATH_INFO'] = env['PATH_INFO'].replace(
-                account_user, account_id, 1)
+                str_to_wsgi(account_user), wsgi_unquote(account_id), 1)
             groups = self._get_user_groups(account, account_user, account_id)
         return groups
@@ -520,7 +523,7 @@ class TempAuth(object):
             if not isinstance(result[key], list):
                 return "Value for key %s must be a list" % json.dumps(key)
             for grantee in result[key]:
-                if not isinstance(grantee, six.string_types):
+                if not isinstance(grantee, str):
                     return "Elements of %s list must be strings" % json.dumps(
                         key)
@@ -569,6 +572,14 @@ class TempAuth(object):
                               % account_user)
             return None
+        if '.reseller_reader' in user_groups and \
+                account not in self.reseller_prefixes and \
+                not self._dot_account(account) and \
+                req.method in ('GET', 'HEAD'):
+            self.logger.debug("User %s has reseller reader authorizing."
+                              % account_user)
+            return None
         if wsgi_to_str(account) in user_groups and \
                 (req.method not in ('DELETE', 'PUT') or container):
             # The user is admin for the account and is not trying to do an
@@ -666,8 +677,6 @@ class TempAuth(object):
             req = Request(env)
             if self.auth_prefix:
                 req.path_info_pop()
-            req.bytes_transferred = '-'
-            req.client_disconnect = False
             if 'x-storage-token' in req.headers and \
                     'x-auth-token' not in req.headers:
                 req.headers['x-auth-token'] = req.headers['x-storage-token']
@@ -677,7 +686,7 @@ class TempAuth(object):
             self.logger.increment('errors')
             start_response('500 Server Error',
                            [('Content-Type', 'text/plain')])
-            return ['Internal server error.\n']
+            return [b'Internal server error.\n']
     def handle_request(self, req):
         """
@@ -707,6 +716,25 @@ class TempAuth(object):
             req.response = handler(req)
         return req.response
+    def _create_new_token(self, memcache_client,
+                          account, account_user, account_id):
+        # Generate new token
+        token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
+        expires = time() + self.token_life
+        groups = self._get_user_groups(account, account_user, account_id)
+        # Save token
+        memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
+        memcache_client.set(memcache_token_key, (expires, groups),
+                            time=float(expires - time()),
+                            raise_on_error=True)
+        # Record the token with the user info for future use.
+        memcache_user_key = \
+            '%s/user/%s' % (self.reseller_prefix, account_user)
+        memcache_client.set(memcache_user_key, token,
+                            time=float(expires - time()),
+                            raise_on_error=True)
+        return token, expires
     def handle_get_token(self, req):
         """
         Handles the various `request for token and service end point(s)` calls.
@@ -771,24 +799,23 @@ class TempAuth(object):
                 key = req.headers.get('x-storage-pass')
         else:
             return HTTPBadRequest(request=req)
+        unauthed_headers = {
+            'Www-Authenticate': 'Swift realm="%s"' % (account or 'unknown'),
+        }
         if not all((account, user, key)):
             self.logger.increment('token_denied')
-            realm = account or 'unknown'
-            return HTTPUnauthorized(request=req, headers={'Www-Authenticate':
-                                                          'Swift realm="%s"' %
-                                                          realm})
+            return HTTPUnauthorized(request=req, headers=unauthed_headers)
         # Authenticate user
+        account = wsgi_to_str(account)
+        user = wsgi_to_str(user)
+        key = wsgi_to_str(key)
         account_user = account + ':' + user
         if account_user not in self.users:
             self.logger.increment('token_denied')
-            auth = 'Swift realm="%s"' % account
-            return HTTPUnauthorized(request=req,
-                                    headers={'Www-Authenticate': auth})
+            return HTTPUnauthorized(request=req, headers=unauthed_headers)
         if self.users[account_user]['key'] != key:
             self.logger.increment('token_denied')
-            auth = 'Swift realm="unknown"'
-            return HTTPUnauthorized(request=req,
-                                    headers={'Www-Authenticate': auth})
+            return HTTPUnauthorized(request=req, headers=unauthed_headers)
         account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
         # Get memcache client
         memcache_client = cache_from_env(req.environ)
@@ -804,8 +831,7 @@ class TempAuth(object):
             cached_auth_data = memcache_client.get(memcache_token_key)
             if cached_auth_data:
                 expires, old_groups = cached_auth_data
-                old_groups = [group.encode('utf8') if six.PY2 else group
-                              for group in old_groups.split(',')]
+                old_groups = [group for group in old_groups.split(',')]
                 new_groups = self._get_user_groups(account, account_user,
                                                    account_id)
@@ -814,19 +840,11 @@ class TempAuth(object):
                     token = candidate_token
         # Create a new token if one didn't exist
         if not token:
-            # Generate new token
-            token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
-            expires = time() + self.token_life
-            groups = self._get_user_groups(account, account_user, account_id)
-            # Save token
-            memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
-            memcache_client.set(memcache_token_key, (expires, groups),
-                                time=float(expires - time()))
-            # Record the token with the user info for future use.
-            memcache_user_key = \
-                '%s/user/%s' % (self.reseller_prefix, account_user)
-            memcache_client.set(memcache_user_key, token,
-                                time=float(expires - time()))
+            try:
+                token, expires = self._create_new_token(
+                    memcache_client, account, account_user, account_id)
+            except MemcacheConnectionError:
+                return HTTPServiceUnavailable(request=req)
         resp = Response(request=req, headers={
             'x-auth-token': token, 'x-storage-token': token,
             'x-auth-token-expires': str(int(expires - time()))})

swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift 2.23.3py3-none-any.whl → 2.35.0py3-none-any.whl