swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/proxy_logging.py

@@ -19,27 +19,36 @@ Logging middleware for the Swift proxy.
 This serves as both the default logging implementation and an example of how
 to plug in your own logging format/method.
 
-The logging format implemented below is as follows:
+The logging format implemented below is as follows::
 
-client_ip remote_addr end_time.datetime method path protocol
-    status_int referer user_agent auth_token bytes_recvd bytes_sent
-    client_etag transaction_id headers request_time source log_info
-    start_time end_time policy_index
+    client_ip remote_addr end_time.datetime method path protocol
+        status_int referer user_agent auth_token bytes_recvd bytes_sent
+        client_etag transaction_id headers request_time source log_info
+        start_time end_time policy_index
 
 These values are space-separated, and each is url-encoded, so that they can
-be separated with a simple .split()
-
-* remote_addr is the contents of the REMOTE_ADDR environment variable, while
-  client_ip is swift's best guess at the end-user IP, extracted variously
-  from the X-Forwarded-For header, X-Cluster-Ip header, or the REMOTE_ADDR
-  environment variable.
-
-* source (swift.source in the WSGI environment) indicates the code
+be separated with a simple ``.split()``.
+
+* ``remote_addr`` is the contents of the REMOTE_ADDR environment variable,
+  while ``client_ip`` is swift's best guess at the end-user IP, extracted
+  variously from the X-Forwarded-For header, X-Cluster-Ip header, or the
+  REMOTE_ADDR environment variable.
+
+* ``status_int`` is the integer part of the ``status`` string passed to this
+  middleware's start_response function, unless the WSGI environment has an item
+  with key ``swift.proxy_logging_status``, in which case the value of that item
+  is used. Other middleware's may set ``swift.proxy_logging_status`` to
+  override the logging of ``status_int``. In either case, the logged
+  ``status_int`` value is forced to 499 if a client disconnect is detected
+  while this middleware is handling a request, or 500 if an exception is caught
+  while handling a request.
+
+* ``source`` (``swift.source`` in the WSGI environment) indicates the code
   that generated the request, such as most middleware. (See below for
   more detail.)
 
-* log_info (swift.log_info in the WSGI environment) is for additional
-  information that could prove quite useful, such as any x-delete-at
+* ``log_info`` (``swift.log_info`` in the WSGI environment) is for additional
+  information that could prove quite useful, such as any ``x-delete-at``
   value or other "behind the scenes" activity that might not
   otherwise be detectable from the plain log information. Code that
   wishes to add additional log information should use code like
@@ -49,6 +58,11 @@ be separated with a simple .split()
 * Values that are missing (e.g. due to a header not being present) or zero
   are generally represented by a single hyphen ('-').
 
+.. note::
+   The message format may be configured using the ``log_msg_template`` option,
+   allowing fields to be added, removed, re-ordered, and even anonymized. For
+   more information, see https://docs.openstack.org/swift/latest/logs.html
+
 The proxy-logging can be used twice in the proxy server's pipeline when there
 is middleware installed that can return custom responses that don't follow the
 standard pipeline to the proxy server.
@@ -57,30 +71,37 @@ For example, with staticweb, the middleware might intercept a request to
 /v1/AUTH_acc/cont/, make a subrequest to the proxy to retrieve
 /v1/AUTH_acc/cont/index.html and, in effect, respond to the client's original
 request using the 2nd request's body. In this instance the subrequest will be
-logged by the rightmost middleware (with a swift.source set) and the outgoing
-request (with body overridden) will be logged by leftmost middleware.
+logged by the rightmost middleware (with a ``swift.source`` set) and the
+outgoing request (with body overridden) will be logged by leftmost middleware.
 
 Requests that follow the normal pipeline (use the same wsgi environment
 throughout) will not be double logged because an environment variable
-(swift.proxy_access_log_made) is checked/set when a log is made.
+(``swift.proxy_access_log_made``) is checked/set when a log is made.
 
-All middleware making subrequests should take care to set swift.source when
+All middleware making subrequests should take care to set ``swift.source`` when
 needed. With the doubled proxy logs, any consumer/processor of swift's proxy
-logs should look at the swift.source field, the rightmost log value, to decide
-if this is a middleware subrequest or not. A log processor calculating
-bandwidth usage will want to only sum up logs with no swift.source.
+logs should look at the ``swift.source`` field, the rightmost log value, to
+decide if this is a middleware subrequest or not. A log processor calculating
+bandwidth usage will want to only sum up logs with no ``swift.source``.
 """
 
+import os
 import time
 
+from swift.common.constraints import valid_api_version
+from swift.common.middleware.catch_errors import enforce_byte_count
+from swift.common.request_helpers import get_log_info
 from swift.common.swob import Request
 from swift.common.utils import (get_logger, get_remote_client,
-                                config_true_value,
+                                config_true_value, reiterate,
+                                close_if_possible, cap_length,
                                 InputProxy, list_from_csv, get_policy_index,
                                 split_path, StrAnonymizer, StrFormatTime,
                                 LogStringFormatter)
 
 from swift.common.storage_policy import POLICIES
+from swift.common.registry import get_sensitive_headers, \
+    get_sensitive_params, register_sensitive_header
 
 
 class ProxyLoggingMiddleware(object):
@@ -90,6 +111,7 @@ class ProxyLoggingMiddleware(object):
 
     def __init__(self, app, conf, logger=None):
         self.app = app
+        self.pid = os.getpid()
         self.log_formatter = LogStringFormatter(default='-', quote=True)
         self.log_msg_template = conf.get(
             'log_msg_template', (
@@ -115,7 +137,7 @@ class ProxyLoggingMiddleware(object):
         self.valid_methods = conf.get(
             'access_log_statsd_valid_http_methods',
             conf.get('log_statsd_valid_http_methods',
-                     'GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS'))
+                     'GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS,UPDATE'))
         self.valid_methods = [m.strip().upper() for m in
                               self.valid_methods.split(',') if m.strip()]
         access_log_conf = {}
@@ -127,9 +149,10 @@ class ProxyLoggingMiddleware(object):
             value = conf.get('access_' + key, conf.get(key, None))
             if value:
                 access_log_conf[key] = value
-        self.access_logger = logger or get_logger(access_log_conf,
-                                                  log_route='proxy-access')
-        self.access_logger.set_statsd_prefix('proxy-server')
+        self.access_logger = logger or get_logger(
+            access_log_conf,
+            log_route=conf.get('access_log_route', 'proxy-access'),
+            statsd_tail_prefix='proxy-server')
         self.reveal_sensitive_prefix = int(
             conf.get('reveal_sensitive_prefix', 16))
         self.check_log_msg_template_validity()
@@ -144,6 +167,8 @@ class ProxyLoggingMiddleware(object):
                                        self.anonymization_salt),
             'remote_addr': StrAnonymizer('4.3.2.1', self.anonymization_method,
                                          self.anonymization_salt),
+            'domain': StrAnonymizer('', self.anonymization_method,
+                                    self.anonymization_salt),
             'path': StrAnonymizer('/', self.anonymization_method,
                                   self.anonymization_salt),
             'referer': StrAnonymizer('ref', self.anonymization_method,
@@ -171,7 +196,10 @@ class ProxyLoggingMiddleware(object):
             'request_time': '0.05',
             'source': '',
             'log_info': '',
-            'policy_index': ''
+            'policy_index': '',
+            'ttfb': '0.05',
+            'pid': '42',
+            'wire_status_int': '200',
         }
         try:
             self.log_formatter.format(self.log_msg_template, **replacements)
@@ -188,12 +216,29 @@ class ProxyLoggingMiddleware(object):
         env['swift.proxy_access_log_made'] = True
 
     def obscure_sensitive(self, value):
-        if value and len(value) > self.reveal_sensitive_prefix:
-            return value[:self.reveal_sensitive_prefix] + '...'
-        return value
+        return cap_length(value, self.reveal_sensitive_prefix)
+
+    def obscure_req(self, req):
+        for header in get_sensitive_headers():
+            if header in req.headers:
+                req.headers[header] = \
+                    self.obscure_sensitive(req.headers[header])
+
+        obscure_params = get_sensitive_params()
+        new_params = []
+        any_obscured = False
+        for k, v in req.params.items():
+            if k in obscure_params:
+                new_params.append((k, self.obscure_sensitive(v)))
+                any_obscured = True
+            else:
+                new_params.append((k, v))
+        if any_obscured:
+            req.params = new_params
 
     def log_request(self, req, status_int, bytes_received, bytes_sent,
-                    start_time, end_time, resp_headers=None):
+                    start_time, end_time, resp_headers=None, ttfb=0,
+                    wire_status_int=None):
         """
         Log a request.
 
@@ -204,7 +249,14 @@ class ProxyLoggingMiddleware(object):
         :param start_time: timestamp request started
         :param end_time: timestamp request completed
         :param resp_headers: dict of the response headers
+        :param ttfb: time to first byte
+        :param wire_status_int: the on the wire status int
         """
+        self.obscure_req(req)
+        domain = req.environ.get('HTTP_HOST',
+                                 req.environ.get('SERVER_NAME', None))
+        if ':' in domain:
+            domain, port = domain.rsplit(':', 1)
         resp_headers = resp_headers or {}
         logged_headers = None
         if self.log_hdrs:
@@ -220,9 +272,8 @@ class ProxyLoggingMiddleware(object):
         duration_time_str = "%.4f" % (end_time - start_time)
         policy_index = get_policy_index(req.headers, resp_headers)
 
-        acc, cont, obj = None, None, None
-        if req.path.startswith('/v1/'):
-            _, acc, cont, obj = split_path(req.path, 1, 4, True)
+        swift_path = req.environ.get('swift.backend_path', req.path)
+        acc, cont, obj = self.get_aco_from_path(swift_path)
 
         replacements = {
             # Time information
@@ -235,6 +286,8 @@ class ProxyLoggingMiddleware(object):
             'remote_addr': StrAnonymizer(req.remote_addr,
                                          self.anonymization_method,
                                          self.anonymization_salt),
+            'domain': StrAnonymizer(domain, self.anonymization_method,
+                                    self.anonymization_salt),
             'path': StrAnonymizer(req.path_qs, self.anonymization_method,
                                   self.anonymization_salt),
             'referer': StrAnonymizer(req.referer, self.anonymization_method,
@@ -259,16 +312,17 @@ class ProxyLoggingMiddleware(object):
                 req.environ.get('SERVER_PROTOCOL'),
             'status_int': status_int,
             'auth_token':
-                self.obscure_sensitive(
-                    req.headers.get('x-auth-token')),
+                req.headers.get('x-auth-token'),
             'bytes_recvd': bytes_received,
             'bytes_sent': bytes_sent,
             'transaction_id': req.environ.get('swift.trans_id'),
             'request_time': duration_time_str,
             'source': req.environ.get('swift.source'),
-            'log_info':
-                ','.join(req.environ.get('swift.log_info', '')),
+            'log_info': get_log_info(req.environ),
             'policy_index': policy_index,
+            'ttfb': ttfb,
+            'pid': self.pid,
+            'wire_status_int': wire_status_int or status_int,
         }
         self.access_logger.info(
             self.log_formatter.format(self.log_msg_template,
@@ -293,21 +347,28 @@ class ProxyLoggingMiddleware(object):
             self.access_logger.update_stats(metric_name_policy + '.xfer',
                                             bytes_received + bytes_sent)
 
+    def get_aco_from_path(self, swift_path):
+        try:
+            version, acc, cont, obj = split_path(swift_path, 1, 4, True)
+            if not valid_api_version(version):
+                raise ValueError
+        except ValueError:
+            acc, cont, obj = None, None, None
+        return acc, cont, obj
+
     def get_metric_name_type(self, req):
-        if req.path.startswith('/v1/'):
-            try:
-                stat_type = [None, 'account', 'container',
-                             'object'][req.path.strip('/').count('/')]
-            except IndexError:
-                stat_type = 'object'
-        else:
-            stat_type = req.environ.get('swift.source')
-        return stat_type
+        swift_path = req.environ.get('swift.backend_path', req.path)
+        acc, cont, obj = self.get_aco_from_path(swift_path)
+        if obj:
+            return 'object'
+        if cont:
+            return 'container'
+        if acc:
+            return 'account'
+        return req.environ.get('swift.source') or 'UNKNOWN'
 
     def statsd_metric_name(self, req, status_int, method):
         stat_type = self.get_metric_name_type(req)
-        if stat_type is None:
-            return None
         stat_method = method if method in self.valid_methods \
             else 'BAD_METHOD'
         return '.'.join((stat_type, stat_method, str(status_int)))
@@ -343,79 +404,80 @@ class ProxyLoggingMiddleware(object):
         def my_start_response(status, headers, exc_info=None):
             start_response_args[0] = (status, list(headers), exc_info)
 
-        def status_int_for_logging(client_disconnect=False, start_status=None):
+        def status_int_for_logging():
             # log disconnected clients as '499' status code
-            if client_disconnect or input_proxy.client_disconnect:
-                ret_status_int = 499
-            elif start_status is None:
-                ret_status_int = int(
-                    start_response_args[0][0].split(' ', 1)[0])
-            else:
-                ret_status_int = start_status
-            return ret_status_int
+            if input_proxy.client_disconnect:
+                return 499
+            return env.get('swift.proxy_logging_status')
 
         def iter_response(iterable):
-            iterator = iter(iterable)
-            try:
-                chunk = next(iterator)
-                while not chunk:
-                    chunk = next(iterator)
-            except StopIteration:
-                chunk = ''
+            iterator = reiterate(iterable)
+            content_length = None
             for h, v in start_response_args[0][1]:
-                if h.lower() in ('content-length', 'transfer-encoding'):
+                if h.lower() == 'content-length':
+                    content_length = int(v)
+                    break
+                elif h.lower() == 'transfer-encoding':
                     break
             else:
-                if not chunk:
-                    start_response_args[0][1].append(('Content-Length', '0'))
-                elif isinstance(iterable, list):
+                if isinstance(iterator, list):
+                    content_length = sum(len(i) for i in iterator)
                     start_response_args[0][1].append(
-                        ('Content-Length', str(sum(len(i) for i in iterable))))
+                        ('Content-Length', str(content_length)))
+
+            req = Request(env)
+            method = self.method_from_req(req)
+            if method == 'HEAD':
+                content_length = 0
+            if content_length is not None:
+                iterator = enforce_byte_count(iterator, content_length)
+
+            wire_status_int = int(start_response_args[0][0].split(' ', 1)[0])
             resp_headers = dict(start_response_args[0][1])
             start_response(*start_response_args[0])
-            req = Request(env)
 
             # Log timing information for time-to-first-byte (GET requests only)
-            method = self.method_from_req(req)
+            ttfb = 0.0
             if method == 'GET':
-                status_int = status_int_for_logging()
                 policy_index = get_policy_index(req.headers, resp_headers)
-                metric_name = self.statsd_metric_name(req, status_int, method)
+                metric_name = self.statsd_metric_name(
+                    req, wire_status_int, method)
                 metric_name_policy = self.statsd_metric_name_policy(
-                    req, status_int, method, policy_index)
+                    req, wire_status_int, method, policy_index)
+                ttfb = time.time() - start_time
                 if metric_name:
-                    self.access_logger.timing_since(
-                        metric_name + '.first-byte.timing', start_time)
+                    self.access_logger.timing(
+                        metric_name + '.first-byte.timing', ttfb * 1000)
                 if metric_name_policy:
-                    self.access_logger.timing_since(
-                        metric_name_policy + '.first-byte.timing', start_time)
+                    self.access_logger.timing(
+                        metric_name_policy + '.first-byte.timing', ttfb * 1000)
 
             bytes_sent = 0
-            client_disconnect = False
             try:
-                while chunk:
+                for chunk in iterator:
                     bytes_sent += len(chunk)
                     yield chunk
-                    chunk = next(iterator)
-            except StopIteration:  # iterator was depleted
-                return
             except GeneratorExit:  # generator was closed before we finished
-                client_disconnect = True
+                env['swift.proxy_logging_status'] = 499
+                raise
+            except Exception:
+                env['swift.proxy_logging_status'] = 500
                 raise
             finally:
-                status_int = status_int_for_logging(client_disconnect)
+                env.setdefault('swift.proxy_logging_status', wire_status_int)
+                status_int = status_int_for_logging()
                 self.log_request(
                     req, status_int, input_proxy.bytes_received, bytes_sent,
-                    start_time, time.time(), resp_headers=resp_headers)
-                close_method = getattr(iterable, 'close', None)
-                if callable(close_method):
-                    close_method()
+                    start_time, time.time(), resp_headers=resp_headers,
+                    ttfb=ttfb, wire_status_int=wire_status_int)
+                close_if_possible(iterator)
 
         try:
             iterable = self.app(env, my_start_response)
         except Exception:
             req = Request(env)
-            status_int = status_int_for_logging(start_status=500)
+            env['swift.proxy_logging_status'] = 500
+            status_int = status_int_for_logging()
             self.log_request(
                 req, status_int, input_proxy.bytes_received, 0, start_time,
                 time.time())
@@ -428,6 +490,12 @@ def filter_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
 
+    # Normally it would be the middleware that uses the header that
+    # would register it, but because there could be 3rd party auth middlewares
+    # that use 'x-auth-token' or 'x-storage-token' we special case it here.
+    register_sensitive_header('x-auth-token')
+    register_sensitive_header('x-storage-token')
+
     def proxy_logger(app):
         return ProxyLoggingMiddleware(app, conf)
     return proxy_logger

swift/common/middleware/ratelimit.py

@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import time
-from swift import gettext_ as _
 
 import eventlet
 
-from swift.common.utils import cache_from_env, get_logger, register_swift_info
+from swift.common.utils import cache_from_env, get_logger
+from swift.common.registry import register_swift_info
 from swift.proxy.controllers.base import get_account_info, get_container_info
 from swift.common.constraints import valid_api_version
 from swift.common.memcached import MemcacheConnectionError
@@ -242,6 +242,10 @@ class RateLimitMiddleware(object):
         if not self.memcache_client:
             return None
 
+        if req.environ.get('swift.ratelimit.handled'):
+            return None
+        req.environ['swift.ratelimit.handled'] = True
+
         try:
             account_info = get_account_info(req.environ, self.app,
                                             swift_source='RL')
@@ -256,7 +260,7 @@ class RateLimitMiddleware(object):
 
         if account_name in self.ratelimit_blacklist or \
                 account_global_ratelimit == 'BLACKLIST':
-            self.logger.error(_('Returning 497 because of blacklisting: %s'),
+            self.logger.error('Returning 497 because of blacklisting: %s',
                               account_name)
             eventlet.sleep(self.BLACK_LIST_SLEEP)
             return Response(status='497 Blacklisted',
@@ -271,18 +275,21 @@ class RateLimitMiddleware(object):
                 if self.log_sleep_time_seconds and \
                         need_to_sleep > self.log_sleep_time_seconds:
                     self.logger.warning(
-                        _("Ratelimit sleep log: %(sleep)s for "
-                          "%(account)s/%(container)s/%(object)s"),
+                        "Ratelimit sleep log: %(sleep)s for "
+                        "%(account)s/%(container)s/%(object)s",
                         {'sleep': need_to_sleep, 'account': account_name,
                          'container': container_name, 'object': obj_name})
                 if need_to_sleep > 0:
                     eventlet.sleep(need_to_sleep)
             except MaxSleepTimeHitError as e:
+                if obj_name:
+                    path = '/'.join((account_name, container_name, obj_name))
+                else:
+                    path = '/'.join((account_name, container_name))
                 self.logger.error(
-                    _('Returning 498 for %(meth)s to %(acc)s/%(cont)s/%(obj)s '
-                      '. Ratelimit (Max Sleep) %(e)s'),
-                    {'meth': req.method, 'acc': account_name,
-                     'cont': container_name, 'obj': obj_name, 'e': str(e)})
+                    'Returning 498 for %(meth)s to %(path)s. '
+                    'Ratelimit (Max Sleep) %(e)s',
+                    {'meth': req.method, 'path': path, 'e': str(e)})
                 error_resp = Response(status='498 Rate Limited',
                                       body='Slow down', request=req)
                 return error_resp
@@ -301,7 +308,7 @@ class RateLimitMiddleware(object):
             self.memcache_client = cache_from_env(env)
         if not self.memcache_client:
             self.logger.warning(
-                _('Warning: Cannot ratelimit without a memcached client'))
+                'Warning: Cannot ratelimit without a memcached client')
             return self.app(env, start_response)
         try:
             version, account, container, obj = req.split_path(1, 4, True)

swift/common/middleware/read_only.py

@@ -12,10 +12,10 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from swift.common.constraints import check_account_format
+from swift.common.constraints import check_account_format, valid_api_version
 from swift.common.swob import HTTPMethodNotAllowed, Request
-from swift.common.utils import get_logger, config_true_value, \
-    register_swift_info
+from swift.common.utils import get_logger, config_true_value
+from swift.common.registry import register_swift_info
 from swift.proxy.controllers.base import get_info
 
 """
@@ -79,7 +79,9 @@ class ReadOnlyMiddleware(object):
             return self.app(env, start_response)
 
         try:
-            version, account, container, obj = req.split_path(1, 4, True)
+            version, account, container, obj = req.split_path(2, 4, True)
+            if not valid_api_version(version):
+                raise ValueError
         except ValueError:
             return self.app(env, start_response)