swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/recon.py

@@ -20,12 +20,13 @@ import time
 from resource import getpagesize
 
 from swift import __version__ as swiftver
-from swift import gettext_ as _
 from swift.common.constraints import check_mount
 from swift.common.storage_policy import POLICIES
 from swift.common.swob import Request, Response
-from swift.common.utils import get_logger, config_true_value, \
-    SWIFT_CONF_FILE, md5_hash_for_file
+from swift.common.utils import get_logger, SWIFT_CONF_FILE, md5_hash_for_file
+from swift.common.recon import RECON_OBJECT_FILE, RECON_CONTAINER_FILE, \
+    RECON_ACCOUNT_FILE, RECON_DRIVE_FILE, RECON_RELINKER_FILE, \
+    DEFAULT_RECON_CACHE_PATH
 
 
 class ReconMiddleware(object):
@@ -35,7 +36,7 @@ class ReconMiddleware(object):
     /recon/load|mem|async... will return various system metrics.
 
     Needs to be added to the pipeline and requires a filter
-    declaration in the object-server.conf:
+    declaration in the [account|container|object]-server conf file:
 
     [filter:recon]
     use = egg:swift#recon
@@ -48,15 +49,17 @@ class ReconMiddleware(object):
         swift_dir = conf.get('swift_dir', '/etc/swift')
         self.logger = get_logger(conf, log_route='recon')
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
+                                         DEFAULT_RECON_CACHE_PATH)
         self.object_recon_cache = os.path.join(self.recon_cache_path,
-                                               'object.recon')
+                                               RECON_OBJECT_FILE)
         self.container_recon_cache = os.path.join(self.recon_cache_path,
-                                                  'container.recon')
+                                                  RECON_CONTAINER_FILE)
         self.account_recon_cache = os.path.join(self.recon_cache_path,
-                                                'account.recon')
+                                                RECON_ACCOUNT_FILE)
         self.drive_recon_cache = os.path.join(self.recon_cache_path,
-                                              'drive.recon')
+                                              RECON_DRIVE_FILE)
+        self.relink_recon_cache = os.path.join(self.recon_cache_path,
+                                               RECON_RELINKER_FILE)
         self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
         self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
 
@@ -66,26 +69,30 @@ class ReconMiddleware(object):
             self.rings.append(os.path.join(swift_dir,
                                            policy.ring_name + '.ring.gz'))
 
-        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
-
-    def _from_recon_cache(self, cache_keys, cache_file, openr=open):
+    def _from_recon_cache(self, cache_keys, cache_file, openr=open,
+                          ignore_missing=False):
         """retrieve values from a recon cache file
 
         :params cache_keys: list of cache items to retrieve
         :params cache_file: cache file to retrieve items from.
         :params openr: open to use [for unittests]
+        :params ignore_missing: Some recon stats are very temporary, in this
+            case it would be better to not log if things are missing.
         :return: dict of cache items and their values or none if not found
         """
         try:
             with openr(cache_file, 'r') as f:
                 recondata = json.load(f)
-                return dict((key, recondata.get(key)) for key in cache_keys)
-        except IOError:
-            self.logger.exception(_('Error reading recon cache file'))
+                return {key: recondata.get(key) for key in cache_keys}
+        except IOError as err:
+            if err.errno == errno.ENOENT and ignore_missing:
+                pass
+            else:
+                self.logger.exception('Error reading recon cache file')
         except ValueError:
-            self.logger.exception(_('Error parsing recon cache file'))
+            self.logger.exception('Error parsing recon cache file')
         except Exception:
-            self.logger.exception(_('Error retrieving recon data'))
+            self.logger.exception('Error retrieving recon data')
         return dict((key, None) for key in cache_keys)
 
     def get_version(self):
@@ -127,7 +134,7 @@ class ReconMiddleware(object):
 
     def get_async_info(self):
         """get # of async pendings"""
-        return self._from_recon_cache(['async_pending'],
+        return self._from_recon_cache(['async_pending', 'async_pending_last'],
                                       self.object_recon_cache)
 
     def get_driveaudit_error(self):
@@ -135,6 +142,13 @@ class ReconMiddleware(object):
         return self._from_recon_cache(['drive_audit_errors'],
                                       self.drive_recon_cache)
 
+    def get_sharding_info(self):
+        """get sharding info"""
+        return self._from_recon_cache(["sharding_stats",
+                                       "sharding_time",
+                                       "sharding_last"],
+                                      self.container_recon_cache)
+
     def get_replication_info(self, recon_type):
         """get replication info"""
         replication_list = ['replication_time',
@@ -154,12 +168,19 @@ class ReconMiddleware(object):
         else:
             return None
 
+    def get_reconstruction_info(self):
+        """get reconstruction info"""
+        reconstruction_list = ['object_reconstruction_last',
+                               'object_reconstruction_time']
+        return self._from_recon_cache(reconstruction_list,
+                                      self.object_recon_cache)
+
     def get_device_info(self):
         """get devices"""
         try:
             return {self.devices: os.listdir(self.devices)}
         except Exception:
-            self.logger.exception(_('Error listing devices'))
+            self.logger.exception('Error listing devices')
             return {self.devices: None}
 
     def get_updater_info(self, recon_type):
@@ -168,7 +189,9 @@ class ReconMiddleware(object):
             return self._from_recon_cache(['container_updater_sweep'],
                                           self.container_recon_cache)
         elif recon_type == 'object':
-            return self._from_recon_cache(['object_updater_sweep'],
+            return self._from_recon_cache(['object_updater_sweep',
+                                           'object_updater_stats',
+                                           'object_updater_last'],
                                           self.object_recon_cache)
         else:
             return None
@@ -255,7 +278,7 @@ class ReconMiddleware(object):
                 except IOError as err:
                     sums[ringfile] = None
                     if err.errno != errno.ENOENT:
-                        self.logger.exception(_('Error reading ringfile'))
+                        self.logger.exception('Error reading ringfile')
         return sums
 
     def get_swift_conf_md5(self):
@@ -265,7 +288,7 @@ class ReconMiddleware(object):
             hexsum = md5_hash_for_file(SWIFT_CONF_FILE)
         except IOError as err:
             if err.errno != errno.ENOENT:
-                self.logger.exception(_('Error reading swift.conf'))
+                self.logger.exception('Error reading swift.conf')
         return {SWIFT_CONF_FILE: hexsum}
 
     def get_quarantine_count(self):
@@ -330,6 +353,14 @@ class ReconMiddleware(object):
 
         return time.time()
 
+    def get_relinker_info(self):
+        """get relinker info, if any"""
+
+        stat_keys = ['devices', 'workers']
+        return self._from_recon_cache(stat_keys,
+                                      self.relink_recon_cache,
+                                      ignore_missing=True)
+
     def GET(self, req):
         root, rcheck, rtype = req.split_path(1, 3, True)
         all_rtypes = ['account', 'container', 'object']
@@ -372,6 +403,12 @@ class ReconMiddleware(object):
             content = self.get_driveaudit_error()
         elif rcheck == "time":
             content = self.get_time()
+        elif rcheck == "sharding":
+            content = self.get_sharding_info()
+        elif rcheck == "relinker":
+            content = self.get_relinker_info()
+        elif rcheck == "reconstruction" and rtype == 'object':
+            content = self.get_reconstruction_info()
         else:
             content = "Invalid path: %s" % req.path
             return Response(request=req, status="404 Not Found",

swift/common/middleware/s3api/acl_handlers.py

@@ -128,9 +128,23 @@ class BaseAclHandler(object):
             raise Exception('No permission to be checked exists')
 
         if resource == 'object':
+            version_id = self.req.params.get('versionId')
+            if version_id is None:
+                query = {}
+            else:
+                query = {'version-id': version_id}
+            if self.req.method == 'HEAD':
+                # This HEAD for ACL is going to also be the definitive response
+                # to the client so we need to include client params. We don't
+                # do this for other client request methods because they may
+                # have invalid combinations of params and headers for a swift
+                # HEAD request.
+                part_number = self.req.params.get('partNumber')
+                if part_number is not None:
+                    query['part-number'] = part_number
             resp = self.req.get_acl_response(app, 'HEAD',
                                              container, obj,
-                                             headers)
+                                             headers, query=query)
             acl = resp.object_acl
         elif resource == 'container':
             resp = self.req.get_acl_response(app, 'HEAD',
@@ -162,8 +176,8 @@ class BaseAclHandler(object):
             try:
                 elem = fromstring(body, ACL.root_tag)
                 acl = ACL.from_elem(
-                    elem, True, self.req.allow_no_owner)
-            except(XMLSyntaxError, DocumentInvalid):
+                    elem, True, self.req.conf.allow_no_owner)
+            except (XMLSyntaxError, DocumentInvalid):
                 raise MalformedACLError()
             except Exception as e:
                 self.logger.error(e)
@@ -243,6 +257,9 @@ class S3AclHandler(BaseAclHandler):
     """
     S3AclHandler: Handler for S3AclController
     """
+    def HEAD(self, app):
+        self._handle_acl(app, 'HEAD', permission='READ_ACP')
+
     def GET(self, app):
         self._handle_acl(app, 'HEAD', permission='READ_ACP')
 
@@ -460,4 +477,9 @@ ACL_MAP = {
     # Initiate Multipart Upload
     ('POST', 'HEAD', 'container'):
     {'Permission': 'WRITE'},
+    # Versioning
+    ('PUT', 'POST', 'container'):
+    {'Permission': 'WRITE'},
+    ('DELETE', 'GET', 'container'):
+    {'Permission': 'WRITE'},
 }

swift/common/middleware/s3api/acl_utils.py

@@ -41,6 +41,11 @@ def swift_acl_translate(acl, group='', user='', xml=False):
     #      ['HTTP_X_CONTAINER_READ', group + ':' + user]]
     swift_acl['private'] = [['X-Container-Write', '.'],
                             ['X-Container-Read', '.']]
+
+    # Swift doesn't have per-object ACLs, so this is best-effort
+    swift_acl['bucket-owner-full-control'] = swift_acl['private']
+    swift_acl['bucket-owner-read'] = swift_acl['private']
+
     if xml:
         # We are working with XML and need to parse it
         try:
@@ -62,7 +67,7 @@ def swift_acl_translate(acl, group='', user='', xml=False):
             else:
                 acl = 'unsupported'
 
-    if acl == 'authenticated-read':
+    if acl in ('authenticated-read', 'log-delivery-write'):
         raise S3NotImplemented()
     elif acl not in swift_acl:
         raise ACLError()

swift/common/middleware/s3api/controllers/__init__.py

@@ -31,6 +31,10 @@ from swift.common.middleware.s3api.controllers.logging import \
     LoggingStatusController
 from swift.common.middleware.s3api.controllers.versioning import \
     VersioningController
+from swift.common.middleware.s3api.controllers.tagging import \
+    TaggingController
+from swift.common.middleware.s3api.controllers.object_lock import \
+    ObjectLockController
 
 __all__ = [
     'Controller',
@@ -47,6 +51,8 @@ __all__ = [
     'LocationController',
     'LoggingStatusController',
     'VersioningController',
+    'TaggingController',
+    'ObjectLockController',
 
     'UnsupportedController',
 ]

swift/common/middleware/s3api/controllers/acl.py

@@ -19,8 +19,9 @@ from swift.common.utils import public
 
 from swift.common.middleware.s3api.exception import ACLError
 from swift.common.middleware.s3api.controllers.base import Controller
-from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
-    MalformedACLError, UnexpectedContent, MissingSecurityHeader
+from swift.common.middleware.s3api.s3response import (
+    HTTPOk, S3NotImplemented, MalformedACLError, UnexpectedContent,
+    MissingSecurityHeader)
 from swift.common.middleware.s3api.etree import Element, SubElement, tostring
 from swift.common.middleware.s3api.acl_utils import swift_acl_translate, \
     XMLNS_XSI