swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/__init__.py

@@ -17,6 +17,10 @@ import re
 from swift.common.wsgi import WSGIContext
 
 
+def app_property(name):
+    return property(lambda self: getattr(self.app, name))
+
+
 class RewriteContext(WSGIContext):
     base_re = None
 

swift/common/middleware/account_quotas.py

@@ -18,10 +18,44 @@
 given account quota (in bytes) is exceeded while DELETE requests are still
 allowed.
 
-``account_quotas`` uses the ``x-account-meta-quota-bytes`` metadata entry to
-store the quota. Write requests to this metadata entry are only permitted for
-resellers. There is no quota limit if ``x-account-meta-quota-bytes`` is not
-set.
+``account_quotas`` uses the following metadata entries to store the account
+quota
+
++---------------------------------------------+-------------------------------+
+|Metadata                                     | Use                           |
++=============================================+===============================+
+| X-Account-Meta-Quota-Bytes (obsoleted)      | Maximum overall bytes stored  |
+|                                             | in account across containers. |
++---------------------------------------------+-------------------------------+
+| X-Account-Quota-Bytes                       | Maximum overall bytes stored  |
+|                                             | in account across containers. |
++---------------------------------------------+-------------------------------+
+| X-Account-Quota-Bytes-Policy-<policyname>   | Maximum overall bytes stored  |
+|                                             | in account across containers, |
+|                                             | for the given policy.         |
++---------------------------------------------+-------------------------------+
+| X-Account-Quota-Count                       | Maximum object count under    |
+|                                             | account.                      |
++---------------------------------------------+-------------------------------+
+| X-Account-Quota-Count-Policy-<policyname>   | Maximum object count under    |
+|                                             | account, for the given policy.|
++---------------------------------------------+-------------------------------+
+
+
+Write requests to those metadata entries are only permitted for resellers.
+There is no overall byte or object count limit set if the corresponding
+metadata entries are not set.
+
+Additionally, account quotas, of type quota-bytes or quota-count, may be set
+for each storage policy, using metadata of the form ``x-account-<quota type>-\
+policy-<policy name>``. Again, only resellers may update these metadata, and
+there will be no limit for a particular policy if the corresponding metadata
+is not set.
+
+.. note::
+   Per-policy quotas need not sum to the overall account quota, and the sum of
+   all :ref:`container_quotas` for a given policy need not sum to the account's
+   policy quota.
 
 The ``account_quotas`` middleware should be added to the pipeline in your
 ``/etc/swift/proxy-server.conf`` file just after any auth middleware.
@@ -54,8 +88,9 @@ account size has been updated.
 
 from swift.common.swob import HTTPForbidden, HTTPBadRequest, \
     HTTPRequestEntityTooLarge, wsgify
-from swift.common.utils import register_swift_info
-from swift.proxy.controllers.base import get_account_info
+from swift.common.registry import register_swift_info
+from swift.common.storage_policy import POLICIES
+from swift.proxy.controllers.base import get_account_info, get_container_info
 
 
 class AccountQuotaMiddleware(object):
@@ -67,12 +102,74 @@ class AccountQuotaMiddleware(object):
     def __init__(self, app, *args, **kwargs):
         self.app = app
 
+    def validate_and_translate_quotas(self, request, quota_type):
+        new_quotas = {}
+        new_quotas[None] = request.headers.get(
+            'X-Account-%s' % quota_type)
+        if request.headers.get(
+                'X-Remove-Account-%s' % quota_type):
+            new_quotas[None] = ''  # X-Remove dominates if both are present
+
+        for policy in POLICIES:
+            tail = 'Account-%s-Policy-%s' % (quota_type, policy.name)
+            if request.headers.get('X-Remove-' + tail):
+                new_quotas[policy.idx] = ''
+            else:
+                quota = request.headers.pop('X-' + tail, None)
+                new_quotas[policy.idx] = quota
+
+        if request.environ.get('reseller_request') is True:
+            if any(quota and not quota.isdigit()
+                    for quota in new_quotas.values()):
+                raise HTTPBadRequest()
+            for idx, quota in new_quotas.items():
+                if idx is None:
+                    hdr = 'X-Account-Sysmeta-%s' % quota_type
+                else:
+                    hdr = 'X-Account-Sysmeta-%s-Policy-%d' % (quota_type, idx)
+                request.headers[hdr] = quota
+        elif any(quota is not None for quota in new_quotas.values()):
+            # deny quota set for non-reseller
+            raise HTTPForbidden()
+
+    def handle_account(self, request):
+        if request.method in ("POST", "PUT"):
+            # Support old meta format
+            for legacy_header in [
+                'X-Account-Meta-Quota-Bytes',
+                'X-Remove-Account-Meta-Quota-Bytes',
+            ]:
+                new_header = legacy_header.replace('-Meta-', '-')
+                legacy_value = request.headers.get(legacy_header)
+                if legacy_value is not None and not \
+                        request.headers.get(new_header):
+                    request.headers[new_header] = legacy_value
+            # account request, so we pay attention to the quotas
+            self.validate_and_translate_quotas(request, "Quota-Bytes")
+            self.validate_and_translate_quotas(request, "Quota-Count")
+        resp = request.get_response(self.app)
+        # Non-resellers can't update quotas, but they *can* see them
+        # Global quotas
+        postfixes = ('Quota-Bytes', 'Quota-Count')
+        for postfix in postfixes:
+            value = resp.headers.get('X-Account-Sysmeta-%s' % postfix)
+            if value:
+                resp.headers['X-Account-%s' % postfix] = value
+
+        # Per policy quotas
+        for policy in POLICIES:
+            infixes = ('Quota-Bytes-Policy', 'Quota-Count-Policy')
+            for infix in infixes:
+                value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
+                    infix, policy.idx))
+                if value:
+                    resp.headers['X-Account-%s-%s' % (
+                        infix, policy.name)] = value
+        return resp
+
     @wsgify
     def __call__(self, request):
 
-        if request.method not in ("POST", "PUT"):
-            return self.app
-
         try:
             ver, account, container, obj = request.split_path(
                 2, 4, rest_with_last=True)
@@ -80,57 +177,125 @@ class AccountQuotaMiddleware(object):
             return self.app
 
         if not container:
-            # account request, so we pay attention to the quotas
-            new_quota = request.headers.get(
-                'X-Account-Meta-Quota-Bytes')
-            remove_quota = request.headers.get(
-                'X-Remove-Account-Meta-Quota-Bytes')
-        else:
-            # container or object request; even if the quota headers are set
-            # in the request, they're meaningless
-            new_quota = remove_quota = None
-
-        if remove_quota:
-            new_quota = 0    # X-Remove dominates if both are present
+            return self.handle_account(request)
+        # container or object request; even if the quota headers are set
+        # in the request, they're meaningless
 
-        if request.environ.get('reseller_request') is True:
-            if new_quota and not new_quota.isdigit():
-                return HTTPBadRequest()
+        if not (request.method == "PUT" and obj):
             return self.app
+        # OK, object PUT
 
-        # deny quota set for non-reseller
-        if new_quota is not None:
-            return HTTPForbidden()
-
-        if request.method == "POST" or not obj:
+        if request.environ.get('reseller_request') is True:
+            # but resellers aren't constrained by quotas :-)
             return self.app
 
+        # Object PUT request
         content_length = (request.content_length or 0)
 
-        account_info = get_account_info(request.environ, self.app)
-        if not account_info or not account_info['bytes']:
+        account_info = get_account_info(request.environ, self.app,
+                                        swift_source='AQ')
+        if not account_info:
             return self.app
+
+        # Check for quota byte violation
         try:
-            quota = int(account_info['meta'].get('quota-bytes', -1))
+            quota = int(
+                account_info["sysmeta"].get(
+                    "quota-bytes", account_info["meta"].get("quota-bytes", -1)
+                )
+            )
         except ValueError:
+            quota = -1
+        if quota >= 0:
+            new_size = int(account_info['bytes']) + content_length
+            if quota < new_size:
+                resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
+                if 'swift.authorize' in request.environ:
+                    orig_authorize = request.environ['swift.authorize']
+
+                    def reject_authorize(*args, **kwargs):
+                        aresp = orig_authorize(*args, **kwargs)
+                        if aresp:
+                            return aresp
+                        return resp
+                    request.environ['swift.authorize'] = reject_authorize
+                else:
+                    return resp
+
+        # Check for quota count violation
+        try:
+            quota = int(account_info['sysmeta'].get('quota-count', -1))
+        except ValueError:
+            quota = -1
+        if quota >= 0:
+            new_count = int(account_info['total_object_count']) + 1
+            if quota < new_count:
+                resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
+                if 'swift.authorize' in request.environ:
+                    orig_authorize = request.environ['swift.authorize']
+
+                    def reject_authorize(*args, **kwargs):
+                        aresp = orig_authorize(*args, **kwargs)
+                        if aresp:
+                            return aresp
+                        return resp
+                    request.environ['swift.authorize'] = reject_authorize
+                else:
+                    return resp
+
+        container_info = get_container_info(request.environ, self.app,
+                                            swift_source='AQ')
+        if not container_info:
             return self.app
-        if quota < 0:
-            return self.app
+        policy_idx = container_info['storage_policy']
 
-        new_size = int(account_info['bytes']) + content_length
-        if quota < new_size:
-            resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
-            if 'swift.authorize' in request.environ:
-                orig_authorize = request.environ['swift.authorize']
+        # Check quota-byte per policy
+        sysmeta_key = 'quota-bytes-policy-%s' % policy_idx
+        try:
+            policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
+        except ValueError:
+            policy_quota = -1
+        if policy_quota >= 0:
+            policy_stats = account_info['storage_policies'].get(policy_idx, {})
+            new_size = int(policy_stats.get('bytes', 0)) + content_length
+            if policy_quota < new_size:
+                resp = HTTPRequestEntityTooLarge(
+                    body='Upload exceeds policy quota.')
+                if 'swift.authorize' in request.environ:
+                    orig_authorize = request.environ['swift.authorize']
+
+                    def reject_authorize(*args, **kwargs):
+                        aresp = orig_authorize(*args, **kwargs)
+                        if aresp:
+                            return aresp
+                        return resp
+                    request.environ['swift.authorize'] = reject_authorize
+                else:
+                    return resp
 
-                def reject_authorize(*args, **kwargs):
-                    aresp = orig_authorize(*args, **kwargs)
-                    if aresp:
-                        return aresp
+        # Check quota-count per policy
+        sysmeta_key = 'quota-count-policy-%s' % policy_idx
+        try:
+            policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
+        except ValueError:
+            policy_quota = -1
+        if policy_quota >= 0:
+            policy_stats = account_info['storage_policies'].get(policy_idx, {})
+            new_size = int(policy_stats.get('object_count', 0)) + 1
+            if policy_quota < new_size:
+                resp = HTTPRequestEntityTooLarge(
+                    body='Upload exceeds policy quota.')
+                if 'swift.authorize' in request.environ:
+                    orig_authorize = request.environ['swift.authorize']
+
+                    def reject_authorize(*args, **kwargs):
+                        aresp = orig_authorize(*args, **kwargs)
+                        if aresp:
+                            return aresp
+                        return resp
+                    request.environ['swift.authorize'] = reject_authorize
+                else:
                     return resp
-                request.environ['swift.authorize'] = reject_authorize
-            else:
-                return resp
 
         return self.app
 

swift/common/middleware/acl.py

@@ -14,8 +14,7 @@
 # limitations under the License.
 
 import json
-import six
-from six.moves.urllib.parse import unquote, urlparse
+from urllib.parse import unquote, urlparse
 
 
 def clean_acl(name, value):
@@ -217,7 +216,7 @@ def parse_acl_v2(data):
     """
     if data is None:
         return None
-    if data is '':
+    if data == '':
         return {}
     try:
         result = json.loads(data)
@@ -294,12 +293,8 @@ def acls_from_account_info(info):
     if not any((admin_members, readwrite_members, readonly_members)):
         return None
 
-    acls = {
+    return {
         'admin': admin_members,
         'read-write': readwrite_members,
         'read-only': readonly_members,
     }
-    if six.PY2:
-        for k in ('admin', 'read-write', 'read-only'):
-            acls[k] = [v.encode('utf8') for v in acls[k]]
-    return acls

swift/common/middleware/backend_ratelimit.py

@@ -0,0 +1,230 @@
+# Copyright (c) 2022 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import time
+
+from swift.common.request_helpers import split_and_validate_path
+from swift.common.swob import Request, HTTPTooManyBackendRequests, \
+    HTTPException
+from swift.common.utils import get_logger, non_negative_float, \
+    EventletRateLimiter, readconf
+
+RATE_LIMITED_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE',
+                        'REPLICATE')
+BACKEND_RATELIMIT_CONFIG_SECTION = 'backend_ratelimit'
+DEFAULT_BACKEND_RATELIMIT_CONF_FILE = 'backend-ratelimit.conf'
+DEFAULT_CONFIG_RELOAD_INTERVAL = 60.0
+DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND = 0.0
+DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER = 1.0
+
+
+class BackendRateLimitMiddleware(object):
+    """
+    Backend rate-limiting middleware.
+
+    Rate-limits requests to backend storage node devices. Each (device, request
+    method) combination is independently rate-limited. All requests with a
+    'GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE' or 'REPLICATE' method are
+    rate limited on a per-device basis by both a method-specific rate and an
+    overall device rate limit.
+
+    If a request would cause the rate-limit to be exceeded for the method
+    and/or device then a response with a 529 status code is returned.
+    """
+    def __init__(self, app, filter_conf, logger=None):
+        self.app = app
+        self.filter_conf = filter_conf
+        self.logger = logger or get_logger(self.filter_conf,
+                                           log_route='backend_ratelimit')
+        self.requests_per_device_rate_buffer = \
+            DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER
+        # map (device, method) -> rate
+        self.requests_per_device_per_second = {}
+        # map (device, method) -> RateLimiter, populated on-demand
+        self.rate_limiters = {}
+
+        # some config options are *only* read from filter conf at startup...
+        default_conf_path = os.path.join(
+            self.filter_conf.get('swift_dir', '/etc/swift'),
+            DEFAULT_BACKEND_RATELIMIT_CONF_FILE)
+        try:
+            self.conf_path = self.filter_conf['backend_ratelimit_conf_path']
+            self.is_config_file_expected = True
+        except KeyError:
+            self.conf_path = default_conf_path
+            self.is_config_file_expected = False
+        self.config_reload_interval = non_negative_float(
+            filter_conf.get('config_reload_interval',
+                            DEFAULT_CONFIG_RELOAD_INTERVAL))
+
+        # other conf options are read from filter section at startup but may
+        # also be overridden by options in a separate config file...
+        self._last_config_reload_attempt = time.time()
+        self._apply_config(self.filter_conf)
+        self._load_config_file()
+
+    def _refresh_ratelimiters(self):
+        # note: if we ever wanted to prune the ratelimiters (in case devices
+        # have been removed) we could inspect each ratelimiter's running_time
+        # and remove those with very old running_time
+        for (dev, method), rl in self.rate_limiters.items():
+            rl.set_max_rate(self.requests_per_device_per_second[method])
+            rl.set_rate_buffer(self.requests_per_device_rate_buffer)
+
+    def _apply_config(self, conf):
+        modified = False
+        reqs_per_device_rate_buffer = non_negative_float(
+            conf.get('requests_per_device_rate_buffer',
+                     DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER))
+
+        # note: 'None' key holds the aggregate per-device limit for all methods
+        reqs_per_device_per_second = {None: non_negative_float(
+            conf.get('requests_per_device_per_second', 0.0))}
+        for method in RATE_LIMITED_METHODS:
+            val = non_negative_float(
+                conf.get('%s_requests_per_device_per_second'
+                         % method.lower(), 0.0))
+            reqs_per_device_per_second[method] = val
+
+        if reqs_per_device_rate_buffer != self.requests_per_device_rate_buffer:
+            self.requests_per_device_rate_buffer = reqs_per_device_rate_buffer
+            modified = True
+        if reqs_per_device_per_second != self.requests_per_device_per_second:
+            self.requests_per_device_per_second = reqs_per_device_per_second
+            self.is_any_rate_limit_configured = any(
+                self.requests_per_device_per_second.values())
+            modified = True
+        if modified:
+            self._refresh_ratelimiters()
+        return modified
+
+    def _load_config_file(self):
+        # If conf file can be read then apply its options to the filter conf
+        # options, discarding *all* options previously loaded from the conf
+        # file i.e. options deleted from the conf file will revert to the
+        # filter conf value or default value. If the conf file cannot be read
+        # or is invalid, then the current config is left unchanged.
+        try:
+            new_conf = dict(self.filter_conf)  # filter_conf not current conf
+            new_conf.update(
+                readconf(self.conf_path, BACKEND_RATELIMIT_CONFIG_SECTION))
+            modified = self._apply_config(new_conf)
+            if modified:
+                self.logger.info('Loaded config file %s, config changed',
+                                 self.conf_path)
+            elif not self.is_config_file_expected:
+                self.logger.info('Loaded new config file %s, config unchanged',
+                                 self.conf_path)
+            else:
+                self.logger.debug(
+                    'Loaded existing config file %s, config unchanged',
+                    self.conf_path)
+            self.is_config_file_expected = True
+        except IOError as err:
+            if self.is_config_file_expected:
+                self.logger.warning(
+                    'Failed to load config file, config unchanged: %s', err)
+            self.is_config_file_expected = False
+        except ValueError as err:
+            # ...but if it exists it should be valid
+            self.logger.warning('Invalid config file %s, config unchanged: %s',
+                                self.conf_path, err)
+
+    def _maybe_reload_config(self):
+        if self.config_reload_interval:
+            now = time.time()
+            if (now - self._last_config_reload_attempt
+                    >= self.config_reload_interval):
+                try:
+                    self._load_config_file()
+                except Exception:  # noqa
+                    self.logger.exception('Error reloading config file')
+                finally:
+                    # always reset last loaded time to avoid re-try storm
+                    self._last_config_reload_attempt = now
+
+    def _get_ratelimiter(self, device, method=None):
+        """
+        Get a rate limiter for the (device, method) combination. If a rate
+        limiter does not yet exist for the given (device, method) combination
+        then it is created and added to the map of rate limiters.
+
+        :param: the device.
+        :method: the request method; if None then the aggregate rate limiter
+            for all requests to the device is returned.
+        :returns: an instance of ``EventletRateLimiter``.
+        """
+        try:
+            rl = self.rate_limiters[(device, method)]
+        except KeyError:
+            rl = EventletRateLimiter(
+                max_rate=self.requests_per_device_per_second[method],
+                rate_buffer=self.requests_per_device_rate_buffer,
+                running_time=time.time(),
+                burst_after_idle=True)
+            self.rate_limiters[(device, method)] = rl
+        return rl
+
+    def _is_allowed(self, device, method):
+        """
+        Evaluate backend rate-limiting policies for the incoming request.
+
+        A request is allowed when neither the per-(device, method) rate-limit
+        nor the per-device rate-limit has been reached.
+
+        Note: a request will be disallowed if the aggregate per-device
+        rate-limit has been reached, even if the per-(device, method)
+        rate-limit has not been reached for the request's method.
+
+        :param: the device.
+        :method: the request method.
+        :returns: boolean, is_allowed.
+        """
+        return (self._get_ratelimiter(device, None).is_allowed()
+                and self._get_ratelimiter(device, method).is_allowed())
+
+    def __call__(self, env, start_response):
+        """
+        WSGI entry point.
+
+        :param env: WSGI environment dictionary
+        :param start_response: WSGI callable
+        """
+        self._maybe_reload_config()
+        req = Request(env)
+        handler = self.app
+        if (self.is_any_rate_limit_configured
+                and req.method in RATE_LIMITED_METHODS):
+            try:
+                device, partition, _ = split_and_validate_path(req, 1, 3, True)
+                int(partition)  # check it's a valid partition
+            except (ValueError, HTTPException):
+                # request may not have device/partition e.g. a healthcheck req
+                pass
+            else:
+                if not self._is_allowed(device, req.method):
+                    self.logger.increment('backend.ratelimit')
+                    handler = HTTPTooManyBackendRequests()
+        return handler(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def backend_ratelimit_filter(app):
+        return BackendRateLimitMiddleware(app, conf)
+
+    return backend_ratelimit_filter