swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/container/server.py

@@ -15,6 +15,7 @@
 
 import json
 import os
+import sys
 import time
 import traceback
 
@@ -38,7 +39,7 @@ from swift.common.utils import get_logger, hash_path, public, \
     config_true_value, timing_stats, replication, \
     override_bytes_from_content_type, get_log_line, \
     config_fallocate_value, fs_has_free_space, list_from_csv, \
-    ShardRange
+    ShardRange, parse_options
 from swift.common.constraints import valid_timestamp, check_utf8, \
     check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.bufferedhttp import http_connect
@@ -53,6 +54,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
     HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \
     HTTPInsufficientStorage, HTTPException, HTTPMovedPermanently, \
     wsgi_to_str, str_to_wsgi
+from swift.common.wsgi import run_wsgi
 
 
 def gen_resp_headers(info, is_deleted=False):
@@ -141,17 +143,7 @@ class ContainerController(BaseStorageServer):
         self.replicator_rpc = ContainerReplicatorRpc(
             self.root, DATADIR, ContainerBroker, self.mount_check,
             logger=self.logger)
-        if conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            self.auto_create_account_prefix = \
-                conf['auto_create_account_prefix']
-        else:
-            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
         self.shards_account_prefix = (
             self.auto_create_account_prefix + 'shards_')
         if config_true_value(conf.get('allow_versions', 'f')):
@@ -511,59 +503,42 @@ class ContainerController(BaseStorageServer):
             return HTTPInsufficientStorage(drive=drive, request=req)
         if not self.check_free_space(drive):
             return HTTPInsufficientStorage(drive=drive, request=req)
-        requested_policy_index = self.get_and_validate_policy_index(req)
-        broker = self._get_container_broker(drive, part, account, container)
-        if obj:     # put container object
-            # obj put expects the policy_index header, default is for
-            # legacy support during upgrade.
-            obj_policy_index = requested_policy_index or 0
-            self._maybe_autocreate(
-                broker, req_timestamp, account, obj_policy_index, req)
-            # redirect if a shard exists for this object name
-            response = self._redirect_to_shard(req, broker, obj)
-            if response:
-                return response
-
-            broker.put_object(obj, req_timestamp.internal,
-                              int(req.headers['x-size']),
-                              wsgi_to_str(req.headers['x-content-type']),
-                              wsgi_to_str(req.headers['x-etag']), 0,
-                              obj_policy_index,
-                              wsgi_to_str(req.headers.get(
-                                  'x-content-type-timestamp')),
-                              wsgi_to_str(req.headers.get('x-meta-timestamp')))
-            return HTTPCreated(request=req)
 
+        broker = self._get_container_broker(drive, part, account, container)
+        if obj:
+            return self.PUT_object(req, broker, account, obj, req_timestamp)
         record_type = req.headers.get('x-backend-record-type', '').lower()
         if record_type == RECORD_TYPE_SHARD:
-            try:
-                # validate incoming data...
-                shard_ranges = [ShardRange.from_dict(sr)
-                                for sr in json.loads(req.body)]
-            except (ValueError, KeyError, TypeError) as err:
-                return HTTPBadRequest('Invalid body: %r' % err)
-            created = self._maybe_autocreate(
-                broker, req_timestamp, account, requested_policy_index, req)
-            self._update_metadata(req, broker, req_timestamp, 'PUT')
-            if shard_ranges:
-                # TODO: consider writing the shard ranges into the pending
-                # file, but if so ensure an all-or-none semantic for the write
-                broker.merge_shard_ranges(shard_ranges)
-        else:   # put container
-            if requested_policy_index is None:
-                # use the default index sent by the proxy if available
-                new_container_policy = req.headers.get(
-                    'X-Backend-Storage-Policy-Default', int(POLICIES.default))
-            else:
-                new_container_policy = requested_policy_index
-            created = self._update_or_create(req, broker,
-                                             req_timestamp.internal,
-                                             new_container_policy,
-                                             requested_policy_index)
-            self._update_metadata(req, broker, req_timestamp, 'PUT')
-            resp = self.account_update(req, account, container, broker)
-            if resp:
-                return resp
+            return self.PUT_shard(req, broker, account, req_timestamp)
+        else:
+            return self.PUT_container(req, broker, account,
+                                      container, req_timestamp)
+
+    @timing_stats()
+    def PUT_object(self, req, broker, account, obj, req_timestamp):
+        """Put object into container."""
+        # obj put expects the policy_index header, default is for
+        # legacy support during upgrade.
+        requested_policy_index = self.get_and_validate_policy_index(req)
+        obj_policy_index = requested_policy_index or 0
+        self._maybe_autocreate(
+            broker, req_timestamp, account, obj_policy_index, req)
+        # redirect if a shard exists for this object name
+        response = self._redirect_to_shard(req, broker, obj)
+        if response:
+            return response
+
+        broker.put_object(obj, req_timestamp.internal,
+                          int(req.headers['x-size']),
+                          wsgi_to_str(req.headers['x-content-type']),
+                          wsgi_to_str(req.headers['x-etag']), 0,
+                          obj_policy_index,
+                          wsgi_to_str(req.headers.get(
+                              'x-content-type-timestamp')),
+                          wsgi_to_str(req.headers.get('x-meta-timestamp')))
+        return HTTPCreated(request=req)
+
+    def _create_ok_resp(self, req, broker, created):
         if created:
             return HTTPCreated(request=req,
                                headers={'x-backend-storage-policy-index':
@@ -573,6 +548,45 @@ class ContainerController(BaseStorageServer):
                                 headers={'x-backend-storage-policy-index':
                                          broker.storage_policy_index})
 
+    @timing_stats()
+    def PUT_shard(self, req, broker, account, req_timestamp):
+        """Put shards into container."""
+        requested_policy_index = self.get_and_validate_policy_index(req)
+        try:
+            # validate incoming data...
+            shard_ranges = [ShardRange.from_dict(sr)
+                            for sr in json.loads(req.body)]
+        except (ValueError, KeyError, TypeError) as err:
+            return HTTPBadRequest('Invalid body: %r' % err)
+        created = self._maybe_autocreate(
+            broker, req_timestamp, account, requested_policy_index, req)
+        self._update_metadata(req, broker, req_timestamp, 'PUT')
+        if shard_ranges:
+            # TODO: consider writing the shard ranges into the pending
+            # file, but if so ensure an all-or-none semantic for the write
+            broker.merge_shard_ranges(shard_ranges)
+        return self._create_ok_resp(req, broker, created)
+
+    @timing_stats()
+    def PUT_container(self, req, broker, account, container, req_timestamp):
+        """Update or create container."""
+        requested_policy_index = self.get_and_validate_policy_index(req)
+        if requested_policy_index is None:
+            # use the default index sent by the proxy if available
+            new_container_policy = req.headers.get(
+                'X-Backend-Storage-Policy-Default', int(POLICIES.default))
+        else:
+            new_container_policy = requested_policy_index
+        created = self._update_or_create(req, broker,
+                                         req_timestamp.internal,
+                                         new_container_policy,
+                                         requested_policy_index)
+        self._update_metadata(req, broker, req_timestamp, 'PUT')
+        resp = self.account_update(req, account, container, broker)
+        if resp:
+            return resp
+        return self._create_ok_resp(req, broker, created)
+
     @public
     @timing_stats(sample_rate=0.1)
     def HEAD(self, req):
@@ -600,29 +614,42 @@ class ContainerController(BaseStorageServer):
         resp.last_modified = Timestamp(headers['X-PUT-Timestamp']).ceil()
         return resp
 
-    def update_data_record(self, record):
+    def update_shard_record(self, record, shard_record_full=True):
+        """
+        Return the shard_range database record as a dict, the keys will depend
+        on the database fields provided in the record.
+
+        :param record: shard entry record, either ShardRange or Namespace.
+        :param shard_record_full: boolean, when true the timestamp field is
+                                  added as "last_modified" in iso format.
+        :returns: dict suitable for listing responses
         """
-        Perform any mutations to container listing records that are common to
-        all serialization formats, and returns it as a dict.
+        response = dict(record)
+        if shard_record_full:
+            created = record.timestamp
+            response['last_modified'] = Timestamp(created).isoformat
+        return response
+
+    def update_object_record(self, record):
+        """
+        Perform mutation to container listing records that are common to all
+        serialization formats, and returns it as a dict.
 
         Converts created time to iso timestamp.
         Replaces size with 'swift_bytes' content type parameter.
 
-        :params record: object entry record
+        :param record: object entry record
         :returns: modified record
         """
-        if isinstance(record, ShardRange):
-            created = record.timestamp
-            response = dict(record)
-        else:
-            (name, created, size, content_type, etag) = record[:5]
-            name_ = name.decode('utf8') if six.PY2 else name
-            if content_type is None:
-                return {'subdir': name_}
-            response = {
-                'bytes': size, 'hash': etag, 'name': name_,
-                'content_type': content_type}
-            override_bytes_from_content_type(response, logger=self.logger)
+        # record is object info
+        (name, created, size, content_type, etag) = record[:5]
+        name_ = name.decode('utf8') if six.PY2 else name
+        if content_type is None:
+            return {'subdir': name_}
+        response = {
+            'bytes': size, 'hash': etag, 'name': name_,
+            'content_type': content_type}
+        override_bytes_from_content_type(response, logger=self.logger)
         response['last_modified'] = Timestamp(created).isoformat
         return response
 
@@ -685,11 +712,11 @@ class ContainerController(BaseStorageServer):
           either the string or integer representation of
           :data:`~swift.common.utils.ShardRange.STATES`.
 
-          Two alias values may be used in a ``states`` parameter value:
-          ``listing`` will cause the listing to include all shard ranges in a
-          state suitable for contributing to an object listing; ``updating``
-          will cause the listing to include all shard ranges in a state
-          suitable to accept an object update.
+          Alias values may be used in a ``states`` parameter value. The
+          ``listing`` alias will cause the listing to include all shard ranges
+          in a state suitable for contributing to an object listing. The
+          ``updating`` alias will cause the listing to include all shard ranges
+          in a state suitable to accept an object update.
 
           If either of these aliases is used then the shard range listing will
           if necessary be extended with a synthesised 'filler' range in order
@@ -698,6 +725,23 @@ class ContainerController(BaseStorageServer):
           uncovered tail of the requested name range and will point back to the
           same container.
 
+          The ``auditing`` alias will cause the listing to include all shard
+          ranges in a state useful to the sharder while auditing a shard
+          container. This alias will not cause a 'filler' range to be added,
+          but will cause the container's own shard range to be included in the
+          listing. For now, ``auditing`` is only supported when
+          'X-Backend-Record-Shard-Format' is 'full'.
+
+        * Shard range listings can be simplified to include only Namespace
+          only attributes (name, lower and upper) if the caller send the header
+          ``X-Backend-Record-Shard-Format`` with value 'namespace' as a hint
+          that it would prefer namespaces. If this header doesn't exist or the
+          value is 'full', the listings will default to include all attributes
+          of shard ranges. But if params has includes/marker/end_marker then
+          the response will be full shard ranges, regardless the header of
+          ``X-Backend-Record-Shard-Format``. The response header
+          ``X-Backend-Record-Type`` will tell the user what type it gets back.
+
         * Listings are not normally returned from a deleted container. However,
           the ``X-Backend-Override-Deleted`` header may be used with a value in
           :attr:`swift.common.utils.TRUE_VALUES` to force a shard range
@@ -709,13 +753,6 @@ class ContainerController(BaseStorageServer):
         """
         drive, part, account, container, obj = get_obj_name_and_placement(req)
         params = validate_container_params(req)
-        path = params.get('path')
-        prefix = params.get('prefix')
-        delimiter = params.get('delimiter')
-        marker = params.get('marker', '')
-        end_marker = params.get('end_marker')
-        limit = params['limit']
-        reverse = config_true_value(params.get('reverse'))
         out_content_type = listing_formats.get_listing_content_type(req)
         try:
             check_drive(self.root, drive, self.mount_check)
@@ -730,70 +767,154 @@ class ContainerController(BaseStorageServer):
         if record_type == 'auto' and db_state in (SHARDING, SHARDED):
             record_type = 'shard'
         if record_type == 'shard':
-            override_deleted = info and config_true_value(
-                req.headers.get('x-backend-override-deleted', False))
-            resp_headers = gen_resp_headers(
-                info, is_deleted=is_deleted and not override_deleted)
-            if is_deleted and not override_deleted:
-                return HTTPNotFound(request=req, headers=resp_headers)
-            resp_headers['X-Backend-Record-Type'] = 'shard'
-            includes = params.get('includes')
-            override_filter_hdr = req.headers.get(
-                'x-backend-override-shard-name-filter', '').lower()
-            if override_filter_hdr == db_state == 'sharded':
-                # respect the request to send back *all* ranges if the db is in
-                # sharded state
-                resp_headers['X-Backend-Override-Shard-Name-Filter'] = 'true'
-                marker = end_marker = includes = None
-                reverse = False
-            states = params.get('states')
-            fill_gaps = include_own = False
-            if states:
-                states = list_from_csv(states)
-                fill_gaps = any(('listing' in states, 'updating' in states))
-                # 'auditing' is used during shard audit; if the shard is
-                # shrinking then it needs to get acceptor shard ranges, which
-                # may be the root container itself, so use include_own
-                include_own = 'auditing' in states
-                try:
-                    states = broker.resolve_shard_range_states(states)
-                except ValueError:
-                    return HTTPBadRequest(request=req, body='Bad state')
-            include_deleted = config_true_value(
-                req.headers.get('x-backend-include-deleted', False))
+            return self.GET_shard(req, broker, container, params, info,
+                                  is_deleted, out_content_type)
+        else:
+            return self.GET_object(req, broker, container, params, info,
+                                   is_deleted, out_content_type)
+
+    @timing_stats()
+    def GET_shard(self, req, broker, container, params, info,
+                  is_deleted, out_content_type):
+        """
+        Returns a list of persisted shard ranges or namespaces in response.
+
+        :param req: swob.Request object
+        :param broker: container DB broker object
+        :param container: container name
+        :param params: the request params.
+        :param info: the global info for the container
+        :param is_deleted: the is_deleted status for the container.
+        :param out_content_type: content type as a string.
+        :returns: an instance of :class:`swift.common.swob.Response`
+        """
+        override_deleted = info and config_true_value(
+            req.headers.get('x-backend-override-deleted', False))
+        resp_headers = gen_resp_headers(
+            info, is_deleted=is_deleted and not override_deleted)
+
+        if is_deleted and not override_deleted:
+            return HTTPNotFound(request=req, headers=resp_headers)
+
+        marker = params.get('marker', '')
+        end_marker = params.get('end_marker')
+        reverse = config_true_value(params.get('reverse'))
+        states = params.get('states')
+        includes = params.get('includes')
+        include_deleted = config_true_value(
+            req.headers.get('x-backend-include-deleted', False))
+
+        resp_headers['X-Backend-Record-Type'] = 'shard'
+        override_filter_hdr = req.headers.get(
+            'x-backend-override-shard-name-filter', '').lower()
+        if override_filter_hdr == info.get('db_state') == 'sharded':
+            # respect the request to send back *all* ranges if the db is in
+            # sharded state
+            resp_headers['X-Backend-Override-Shard-Name-Filter'] = 'true'
+            marker = end_marker = includes = None
+            reverse = False
+        fill_gaps = include_own = False
+        if states:
+            states = list_from_csv(states)
+            fill_gaps = any(('listing' in states, 'updating' in states))
+            # The 'auditing' state alias is used by the sharder during
+            # shard audit; if the shard is shrinking then it needs to get
+            # acceptor shard ranges, which may be the root container
+            # itself, so use include_own.
+            include_own = 'auditing' in states
+            try:
+                states = broker.resolve_shard_range_states(states)
+            except ValueError:
+                return HTTPBadRequest(request=req, body='Bad state')
+
+        # For record type of 'shard', user can specify an additional header
+        # to ask for list of Namespaces instead of full ShardRanges.
+        # This will allow proxy server who is going to retrieve Namespace
+        # to talk to older version of container servers who don't support
+        # Namespace yet during upgrade.
+        shard_format = req.headers.get(
+            'x-backend-record-shard-format', 'full').lower()
+        if shard_format == 'namespace':
+            resp_headers['X-Backend-Record-Shard-Format'] = 'namespace'
+            # Namespace GET does not support all the options of Shard Range
+            # GET: 'x-backend-include-deleted' cannot be supported because
+            # there is no way for a Namespace to indicate the deleted state;
+            # the 'auditing' state query parameter is not supported because it
+            # is specific to the sharder which only requests full shard ranges.
+            if include_deleted:
+                return HTTPBadRequest(
+                    request=req, body='No include_deleted for namespace GET')
+            if include_own:
+                return HTTPBadRequest(
+                    request=req, body='No auditing state for namespace GET')
+            shard_format_full = False
+            container_list = broker.get_namespaces(
+                marker, end_marker, includes, reverse, states, fill_gaps)
+        else:
+            resp_headers['X-Backend-Record-Shard-Format'] = 'full'
+            shard_format_full = True
             container_list = broker.get_shard_ranges(
                 marker, end_marker, includes, reverse, states=states,
                 include_deleted=include_deleted, fill_gaps=fill_gaps,
                 include_own=include_own)
-        else:
-            requested_policy_index = self.get_and_validate_policy_index(req)
-            resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
-            if is_deleted:
-                return HTTPNotFound(request=req, headers=resp_headers)
-            resp_headers['X-Backend-Record-Type'] = 'object'
-            storage_policy_index = (
-                requested_policy_index if requested_policy_index is not None
-                else info['storage_policy_index'])
-            resp_headers['X-Backend-Record-Storage-Policy-Index'] = \
-                storage_policy_index
-            # Use the retired db while container is in process of sharding,
-            # otherwise use current db
-            src_broker = broker.get_brokers()[0]
-            container_list = src_broker.list_objects_iter(
-                limit, marker, end_marker, prefix, delimiter, path,
-                storage_policy_index=storage_policy_index,
-                reverse=reverse, allow_reserved=req.allow_reserved_names)
-        return self.create_listing(req, out_content_type, info, resp_headers,
-                                   broker.metadata, container_list, container)
-
-    def create_listing(self, req, out_content_type, info, resp_headers,
-                       metadata, container_list, container):
+        listing = [self.update_shard_record(record, shard_format_full)
+                   for record in container_list]
+        return self._create_GET_response(req, out_content_type, info,
+                                         resp_headers, broker.metadata,
+                                         container, listing)
+
+    @timing_stats()
+    def GET_object(self, req, broker, container, params, info,
+                   is_deleted, out_content_type):
+        """
+        Returns a list of objects in response.
+
+        :param req: swob.Request object
+        :param broker: container DB broker object
+        :param container: container name
+        :param params: the request params.
+        :param info: the global info for the container
+        :param is_deleted: the is_deleted status for the container.
+        :param out_content_type: content type as a string.
+        :returns: an instance of :class:`swift.common.swob.Response`
+        """
+        marker = params.get('marker', '')
+        end_marker = params.get('end_marker')
+        reverse = config_true_value(params.get('reverse'))
+        path = params.get('path')
+        prefix = params.get('prefix')
+        delimiter = params.get('delimiter')
+        limit = params['limit']
+        requested_policy_index = self.get_and_validate_policy_index(req)
+        resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
+        if is_deleted:
+            return HTTPNotFound(request=req, headers=resp_headers)
+        resp_headers['X-Backend-Record-Type'] = 'object'
+        storage_policy_index = (
+            requested_policy_index if requested_policy_index is not None
+            else info['storage_policy_index'])
+        resp_headers['X-Backend-Record-Storage-Policy-Index'] = \
+            storage_policy_index
+        # Use the retired db while container is in process of sharding,
+        # otherwise use current db
+        src_broker = broker.get_brokers()[0]
+        container_list = src_broker.list_objects_iter(
+            limit, marker, end_marker, prefix, delimiter, path,
+            storage_policy_index=storage_policy_index,
+            reverse=reverse, allow_reserved=req.allow_reserved_names)
+        listing = [self.update_object_record(record)
+                   for record in container_list]
+        return self._create_GET_response(req, out_content_type, info,
+                                         resp_headers, broker.metadata,
+                                         container, listing)
+
+    def _create_GET_response(self, req, out_content_type, info, resp_headers,
+                             metadata, container, listing):
         for key, (value, _timestamp) in metadata.items():
             if value and (key.lower() in self.save_headers or
                           is_sys_or_user_meta('container', key)):
                 resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)
-        listing = [self.update_data_record(record)
-                   for record in container_list]
+
         if out_content_type.endswith('/xml'):
             body = listing_formats.container_to_xml(listing, container)
         elif out_content_type.endswith('/json'):
@@ -929,3 +1050,12 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return ContainerController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'container-server', **options))
+
+
+if __name__ == '__main__':
+    main()