swift 2.32.1__py2.py3-none-any.whl → 2.33.1__py2.py3-none-any.whl

swift/common/middleware/proxy_logging.py

@@ -27,19 +27,28 @@ The logging format implemented below is as follows::
         start_time end_time policy_index
 
 These values are space-separated, and each is url-encoded, so that they can
-be separated with a simple .split()
-
-* remote_addr is the contents of the REMOTE_ADDR environment variable, while
-  client_ip is swift's best guess at the end-user IP, extracted variously
-  from the X-Forwarded-For header, X-Cluster-Ip header, or the REMOTE_ADDR
-  environment variable.
-
-* source (swift.source in the WSGI environment) indicates the code
+be separated with a simple ``.split()``.
+
+* ``remote_addr`` is the contents of the REMOTE_ADDR environment variable,
+  while ``client_ip`` is swift's best guess at the end-user IP, extracted
+  variously from the X-Forwarded-For header, X-Cluster-Ip header, or the
+  REMOTE_ADDR environment variable.
+
+* ``status_int`` is the integer part of the ``status`` string passed to this
+  middleware's start_response function, unless the WSGI environment has an item
+  with key ``swift.proxy_logging_status``, in which case the value of that item
+  is used. Other middleware's may set ``swift.proxy_logging_status`` to
+  override the logging of ``status_int``. In either case, the logged
+  ``status_int`` value is forced to 499 if a client disconnect is detected
+  while this middleware is handling a request, or 500 if an exception is caught
+  while handling a request.
+
+* ``source`` (``swift.source`` in the WSGI environment) indicates the code
   that generated the request, such as most middleware. (See below for
   more detail.)
 
-* log_info (swift.log_info in the WSGI environment) is for additional
-  information that could prove quite useful, such as any x-delete-at
+* ``log_info`` (``swift.log_info`` in the WSGI environment) is for additional
+  information that could prove quite useful, such as any ``x-delete-at``
   value or other "behind the scenes" activity that might not
   otherwise be detectable from the plain log information. Code that
   wishes to add additional log information should use code like
@@ -62,18 +71,18 @@ For example, with staticweb, the middleware might intercept a request to
 /v1/AUTH_acc/cont/, make a subrequest to the proxy to retrieve
 /v1/AUTH_acc/cont/index.html and, in effect, respond to the client's original
 request using the 2nd request's body. In this instance the subrequest will be
-logged by the rightmost middleware (with a swift.source set) and the outgoing
-request (with body overridden) will be logged by leftmost middleware.
+logged by the rightmost middleware (with a ``swift.source`` set) and the
+outgoing request (with body overridden) will be logged by leftmost middleware.
 
 Requests that follow the normal pipeline (use the same wsgi environment
 throughout) will not be double logged because an environment variable
-(swift.proxy_access_log_made) is checked/set when a log is made.
+(``swift.proxy_access_log_made``) is checked/set when a log is made.
 
-All middleware making subrequests should take care to set swift.source when
+All middleware making subrequests should take care to set ``swift.source`` when
 needed. With the doubled proxy logs, any consumer/processor of swift's proxy
-logs should look at the swift.source field, the rightmost log value, to decide
-if this is a middleware subrequest or not. A log processor calculating
-bandwidth usage will want to only sum up logs with no swift.source.
+logs should look at the ``swift.source`` field, the rightmost log value, to
+decide if this is a middleware subrequest or not. A log processor calculating
+bandwidth usage will want to only sum up logs with no ``swift.source``.
 """
 
 import os
@@ -389,11 +398,11 @@ class ProxyLoggingMiddleware(object):
         def my_start_response(status, headers, exc_info=None):
             start_response_args[0] = (status, list(headers), exc_info)
 
-        def status_int_for_logging(start_status, client_disconnect=False):
+        def status_int_for_logging():
             # log disconnected clients as '499' status code
-            if client_disconnect or input_proxy.client_disconnect:
+            if input_proxy.client_disconnect:
                 return 499
-            return start_status
+            return env.get('swift.proxy_logging_status')
 
         def iter_response(iterable):
             iterator = reiterate(iterable)
@@ -438,21 +447,19 @@ class ProxyLoggingMiddleware(object):
                         metric_name_policy + '.first-byte.timing', ttfb * 1000)
 
             bytes_sent = 0
-            client_disconnect = False
-            start_status = wire_status_int
             try:
                 for chunk in iterator:
                     bytes_sent += len(chunk)
                     yield chunk
             except GeneratorExit:  # generator was closed before we finished
-                client_disconnect = True
+                env['swift.proxy_logging_status'] = 499
                 raise
             except Exception:
-                start_status = 500
+                env['swift.proxy_logging_status'] = 500
                 raise
             finally:
-                status_int = status_int_for_logging(
-                    start_status, client_disconnect)
+                env.setdefault('swift.proxy_logging_status', wire_status_int)
+                status_int = status_int_for_logging()
                 self.log_request(
                     req, status_int, input_proxy.bytes_received, bytes_sent,
                     start_time, time.time(), resp_headers=resp_headers,
@@ -463,7 +470,8 @@ class ProxyLoggingMiddleware(object):
             iterable = self.app(env, my_start_response)
         except Exception:
             req = Request(env)
-            status_int = status_int_for_logging(500)
+            env['swift.proxy_logging_status'] = 500
+            status_int = status_int_for_logging()
             self.log_request(
                 req, status_int, input_proxy.bytes_received, 0, start_time,
                 time.time())

swift/common/middleware/s3api/acl_handlers.py

@@ -168,7 +168,7 @@ class BaseAclHandler(object):
                 elem = fromstring(body, ACL.root_tag)
                 acl = ACL.from_elem(
                     elem, True, self.req.conf.allow_no_owner)
-            except(XMLSyntaxError, DocumentInvalid):
+            except (XMLSyntaxError, DocumentInvalid):
                 raise MalformedACLError()
             except Exception as e:
                 self.logger.error(e)

swift/common/middleware/s3api/controllers/__init__.py

@@ -33,6 +33,8 @@ from swift.common.middleware.s3api.controllers.versioning import \
     VersioningController
 from swift.common.middleware.s3api.controllers.tagging import \
     TaggingController
+from swift.common.middleware.s3api.controllers.object_lock import \
+    ObjectLockController
 
 __all__ = [
     'Controller',
@@ -50,6 +52,7 @@ __all__ = [
     'LoggingStatusController',
     'VersioningController',
     'TaggingController',
+    'ObjectLockController',
 
     'UnsupportedController',
 ]

swift/common/middleware/s3api/controllers/acl.py

@@ -19,8 +19,9 @@ from swift.common.utils import public
 
 from swift.common.middleware.s3api.exception import ACLError
 from swift.common.middleware.s3api.controllers.base import Controller
-from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented,\
-    MalformedACLError, UnexpectedContent, MissingSecurityHeader
+from swift.common.middleware.s3api.s3response import (
+    HTTPOk, S3NotImplemented, MalformedACLError, UnexpectedContent,
+    MissingSecurityHeader)
 from swift.common.middleware.s3api.etree import Element, SubElement, tostring
 from swift.common.middleware.s3api.acl_utils import swift_acl_translate, \
     XMLNS_XSI

swift/common/middleware/s3api/controllers/logging.py

@@ -18,8 +18,8 @@ from swift.common.utils import public
 from swift.common.middleware.s3api.controllers.base import Controller, \
     bucket_operation
 from swift.common.middleware.s3api.etree import Element, tostring
-from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented,\
-    NoLoggingStatusForKey
+from swift.common.middleware.s3api.s3response import (
+    HTTPOk, S3NotImplemented, NoLoggingStatusForKey)
 
 
 class LoggingStatusController(Controller):

swift/common/middleware/s3api/controllers/multi_upload.py

@@ -68,8 +68,9 @@ import time
 import six
 
 from swift.common import constraints
-from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, wsgi_to_str
-from swift.common.utils import json, public, reiterate, md5
+from swift.common.swob import Range, bytes_to_wsgi, normalize_etag, \
+    wsgi_to_str
+from swift.common.utils import json, public, reiterate, md5, Timestamp
 from swift.common.db import utf8encode
 from swift.common.request_helpers import get_container_update_override_key, \
     get_param
@@ -82,7 +83,7 @@ from swift.common.middleware.s3api.s3response import InvalidArgument, \
     ErrorResponse, MalformedXML, BadDigest, KeyTooLongError, \
     InvalidPart, BucketAlreadyExists, EntityTooSmall, InvalidPartOrder, \
     InvalidRequest, HTTPOk, HTTPNoContent, NoSuchKey, NoSuchUpload, \
-    NoSuchBucket, BucketAlreadyOwnedByYou
+    NoSuchBucket, BucketAlreadyOwnedByYou, ServiceUnavailable
 from swift.common.middleware.s3api.utils import unique_id, \
     MULTIUPLOAD_SUFFIX, S3Timestamp, sysmeta_header
 from swift.common.middleware.s3api.etree import Element, SubElement, \
@@ -96,6 +97,17 @@ MAX_COMPLETE_UPLOAD_BODY_SIZE = 2048 * 1024
 
 
 def _get_upload_info(req, app, upload_id):
+    """
+    Make a HEAD request for existing upload object metadata. Tries the upload
+    marker first, and then falls back to the manifest object.
+
+    :param req: an S3Request object.
+    :param app: the wsgi app.
+    :param upload_id: the upload id.
+    :returns: a tuple of (S3Response, boolean) where the boolean is True if the
+        response is from the upload marker and False otherwise.
+    :raises: NoSuchUpload if neither the marker nor the manifest were found.
+    """
 
     container = req.container_name + MULTIUPLOAD_SUFFIX
     obj = '%s/%s' % (req.object_name, upload_id)
@@ -106,14 +118,17 @@ def _get_upload_info(req, app, upload_id):
     # it off for now...
     copy_source = req.headers.pop('X-Amz-Copy-Source', None)
     try:
-        return req.get_response(app, 'HEAD', container=container, obj=obj)
+        resp = req.get_response(app, 'HEAD', container=container, obj=obj)
+        return resp, True
     except NoSuchKey:
+        # ensure consistent path and policy are logged despite manifest HEAD
         upload_marker_path = req.environ.get('s3api.backend_path')
+        policy_index = req.policy_index
         try:
             resp = req.get_response(app, 'HEAD')
             if resp.sysmeta_headers.get(sysmeta_header(
                     'object', 'upload-id')) == upload_id:
-                return resp
+                return resp, False
         except NoSuchKey:
             pass
         finally:
@@ -121,6 +136,8 @@ def _get_upload_info(req, app, upload_id):
             # path, so put it back
             if upload_marker_path is not None:
                 req.environ['s3api.backend_path'] = upload_marker_path
+            if policy_index is not None:
+                req.policy_index = policy_index
         raise NoSuchUpload(upload_id=upload_id)
     finally:
         # ...making sure to restore any copy-source before returning
@@ -651,7 +668,14 @@ class UploadController(Controller):
         Handles Complete Multipart Upload.
         """
         upload_id = get_param(req, 'uploadId')
-        resp = _get_upload_info(req, self.app, upload_id)
+        resp, is_marker = _get_upload_info(req, self.app, upload_id)
+        if (is_marker and
+                resp.sw_headers.get('X-Backend-Timestamp') >= Timestamp.now()):
+            # Somehow the marker was created in the future w.r.t. this thread's
+            # clock. The manifest PUT may succeed but the subsequent marker
+            # DELETE will fail, so don't attempt either.
+            raise ServiceUnavailable
+
         headers = {'Accept': 'application/json',
                    sysmeta_header('object', 'upload-id'): upload_id}
         for key, val in resp.headers.items():

swift/common/middleware/s3api/controllers/object_lock.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2023 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.common.utils import public
+
+from swift.common.middleware.s3api.controllers.base import Controller, \
+    bucket_operation, S3NotImplemented
+from swift.common.middleware.s3api.s3response import \
+    ObjectLockConfigurationNotFoundError
+
+
+class ObjectLockController(Controller):
+    """
+    Handles GET object-lock request, which always returns
+    <ObjectLockEnabled>Disabled</ObjectLockEnabled>
+    """
+    @public
+    @bucket_operation
+    def GET(self, req):
+        """
+        Handles GET object-lock param calls.
+        """
+        raise ObjectLockConfigurationNotFoundError(req.container_name)
+
+    @public
+    @bucket_operation
+    def PUT(self, req):
+        """
+        Handles PUT object-lock param calls.
+        """
+        # Basically we don't support it, so return a 501
+        raise S3NotImplemented('The requested resource is not implemented')

swift/common/middleware/s3api/s3api.py

@@ -366,6 +366,7 @@ class S3ApiMiddleware(object):
 
         if 's3api.backend_path' in env and 'swift.backend_path' not in env:
             env['swift.backend_path'] = env['s3api.backend_path']
+
         return resp(env, start_response)
 
     def handle_request(self, req):
@@ -390,6 +391,9 @@ class S3ApiMiddleware(object):
             raise MethodNotAllowed(req.method,
                                    req.controller.resource_type())
 
+        if req.policy_index is not None:
+            res.headers.setdefault('X-Backend-Storage-Policy-Index',
+                                   req.policy_index)
         return res
 
     def check_pipeline(self, wsgi_conf):

swift/common/middleware/s3api/s3request.py

@@ -26,7 +26,7 @@ from six.moves.urllib.parse import quote, unquote, parse_qsl
 import string
 
 from swift.common.utils import split_path, json, close_if_possible, md5, \
-    streq_const_time
+    streq_const_time, get_policy_index
 from swift.common.registry import get_swift_info
 from swift.common import swob
 from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
@@ -47,7 +47,7 @@ from swift.common.middleware.s3api.controllers import ServiceController, \
     LocationController, LoggingStatusController, PartController, \
     UploadController, UploadsController, VersioningController, \
     UnsupportedController, S3AclController, BucketController, \
-    TaggingController
+    TaggingController, ObjectLockController
 from swift.common.middleware.s3api.s3response import AccessDenied, \
     InvalidArgument, InvalidDigest, BucketAlreadyOwnedByYou, \
     RequestTimeTooSkewed, S3Response, SignatureDoesNotMatch, \
@@ -74,7 +74,8 @@ ALLOWED_SUB_RESOURCES = sorted([
     'versionId', 'versioning', 'versions', 'website',
     'response-cache-control', 'response-content-disposition',
     'response-content-encoding', 'response-content-language',
-    'response-content-type', 'response-expires', 'cors', 'tagging', 'restore'
+    'response-content-type', 'response-expires', 'cors', 'tagging', 'restore',
+    'object-lock'
 ])
 
 
@@ -103,6 +104,7 @@ def _header_acl_property(resource):
     """
     Set and retrieve the acl in self.headers
     """
+
     def getter(self):
         return getattr(self, '_%s' % resource)
 
@@ -121,6 +123,7 @@ class HashingInput(object):
     """
     wsgi.input wrapper to verify the hash of the input as it's read.
     """
+
     def __init__(self, reader, content_length, hasher, expected_hex_hash):
         self._input = reader
         self._to_read = content_length
@@ -549,6 +552,7 @@ class S3Request(swob.Request):
         }
         self.account = None
         self.user_id = None
+        self.policy_index = None
 
         # Avoids that swift.swob.Response replaces Location header value
         # by full URL when absolute path given. See swift.swob for more detail.
@@ -919,8 +923,6 @@ class S3Request(swob.Request):
         src_resp = self.get_response(app, 'HEAD', src_bucket,
                                      swob.str_to_wsgi(src_obj),
                                      headers=headers, query=query)
-        # we can't let this HEAD req spoil our COPY
-        self.headers.pop('x-backend-storage-policy-index')
         if src_resp.status_int == 304:  # pylint: disable-msg=E1101
             raise PreconditionFailed()
 
@@ -1051,6 +1053,8 @@ class S3Request(swob.Request):
             return VersioningController
         if 'tagging' in self.params:
             return TaggingController
+        if 'object-lock' in self.params:
+            return ObjectLockController
 
         unsupported = ('notification', 'policy', 'requestPayment', 'torrent',
                        'website', 'cors', 'restore')
@@ -1365,11 +1369,11 @@ class S3Request(swob.Request):
                                             2, 3, True)
             # Update s3.backend_path from the response environ
             self.environ['s3api.backend_path'] = sw_resp.environ['PATH_INFO']
-            # Propogate backend headers back into our req headers for logging
-            for k, v in sw_req.headers.items():
-                if k.lower().startswith('x-backend-'):
-                    self.headers.setdefault(k, v)
 
+        # keep a record of the backend policy index so that the s3api can add
+        # it to the headers of whatever response it returns, which may not
+        # necessarily be this resp.
+        self.policy_index = get_policy_index(sw_req.headers, sw_resp.headers)
         resp = S3Response.from_swift_resp(sw_resp)
         status = resp.status_int  # pylint: disable-msg=E1101
 
@@ -1428,8 +1432,10 @@ class S3Request(swob.Request):
                 raise SlowDown(status='429 Slow Down')
             raise SlowDown()
         if resp.status_int == HTTP_CONFLICT:
-            # TODO: validate that this actually came up out of SLO
-            raise BrokenMPU()
+            if self.method == 'GET':
+                raise BrokenMPU()
+            else:
+                raise ServiceUnavailable()
 
         raise InternalError('unexpected status code %d' % status)
 
@@ -1497,7 +1503,7 @@ class S3Request(swob.Request):
 
             _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
                                             2, 3, True)
-        sw_req = self.to_swift_req(app, self.container_name, None)
+        sw_req = self.to_swift_req('TEST', self.container_name, None)
         info = get_container_info(sw_req.environ, app, swift_source='S3')
         if is_success(info['status']):
             return info
@@ -1536,6 +1542,7 @@ class S3AclRequest(S3Request):
     """
     S3Acl request object.
     """
+
     def __init__(self, env, app=None, conf=None):
         super(S3AclRequest, self).__init__(env, app, conf)
         self.authenticate(app)

swift/common/middleware/s3api/s3response.py

@@ -64,7 +64,7 @@ def translate_swift_to_s3(key, val):
 
     if _key.startswith('x-object-meta-'):
         return translate_meta_key(_key), val
-    elif _key in ('content-length', 'content-type',
+    elif _key in ('accept-ranges', 'content-length', 'content-type',
                   'content-range', 'content-encoding',
                   'content-disposition', 'content-language',
                   'etag', 'last-modified', 'x-robots-tag',
@@ -111,6 +111,7 @@ class S3Response(S3ResponseBase, swob.Response):
     headers instead of Swift's HeaderKeyDict.  This also translates Swift
     specific headers to S3 headers.
     """
+
     def __init__(self, *args, **kwargs):
         swob.Response.__init__(self, *args, **kwargs)
 
@@ -239,7 +240,7 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
         self.info = kwargs.copy()
         for reserved_key in ('headers', 'body'):
             if self.info.get(reserved_key):
-                del(self.info[reserved_key])
+                del (self.info[reserved_key])
 
         swob.HTTPException.__init__(
             self, status=kwargs.pop('status', self._status),
@@ -613,6 +614,16 @@ class NoSuchKey(ErrorResponse):
         ErrorResponse.__init__(self, msg, key=key, *args, **kwargs)
 
 
+class ObjectLockConfigurationNotFoundError(ErrorResponse):
+    _status = '404 Not found'
+    _msg = 'Object Lock configuration does not exist for this bucket'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        if not bucket:
+            raise InternalError()
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
 class NoSuchLifecycleConfiguration(ErrorResponse):
     _status = '404 Not Found'
     _msg = 'The lifecycle configuration does not exist. .'

swift/common/middleware/s3api/utils.py

@@ -113,7 +113,7 @@ class S3Timestamp(utils.Timestamp):
 
     @property
     def s3xmlformat(self):
-        dt = datetime.datetime.utcfromtimestamp(self.ceil())
+        dt = datetime.datetime.fromtimestamp(self.ceil(), utils.UTC)
         return dt.strftime(self.S3_XML_FORMAT)
 
     @classmethod