swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/account/server.py

@@ -15,9 +15,9 @@
 
 import json
 import os
+import sys
 import time
 import traceback
-from swift import gettext_ as _
 
 from eventlet import Timeout
 
@@ -26,12 +26,14 @@ from swift.account.backend import AccountBroker, DATADIR
 from swift.account.utils import account_listing_response, get_response_headers
 from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists
 from swift.common.request_helpers import get_param, \
-    split_and_validate_path
+    split_and_validate_path, validate_internal_account, \
+    validate_internal_container, constrain_req_limit
 from swift.common.utils import get_logger, hash_path, public, \
     Timestamp, storage_directory, config_true_value, \
     timing_stats, replication, get_log_line, \
-    config_fallocate_value, fs_has_free_space
-from swift.common.constraints import valid_timestamp, check_utf8, check_drive
+    config_fallocate_value, fs_has_free_space, parse_options
+from swift.common.constraints import valid_timestamp, check_utf8, \
+    check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common import constraints
 from swift.common.db_replicator import ReplicatorRpc
 from swift.common.base_storage_server import BaseStorageServer
@@ -42,6 +44,33 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
     HTTPPreconditionFailed, HTTPConflict, Request, \
     HTTPInsufficientStorage, HTTPException, wsgi_to_str
 from swift.common.request_helpers import is_sys_or_user_meta
+from swift.common.wsgi import run_wsgi
+
+
+def get_account_name_and_placement(req):
+    """
+    Split and validate path for an account.
+
+    :param req: a swob request
+
+    :returns: a tuple of path parts as strings
+    """
+    drive, part, account = split_and_validate_path(req, 3)
+    validate_internal_account(account)
+    return drive, part, account
+
+
+def get_container_name_and_placement(req):
+    """
+    Split and validate path for a container.
+
+    :param req: a swob request
+
+    :returns: a tuple of path parts as strings
+    """
+    drive, part, account, container = split_and_validate_path(req, 3, 4)
+    validate_internal_container(account, container)
+    return drive, part, account, container
 
 
 class AccountController(BaseStorageServer):
@@ -58,10 +87,12 @@ class AccountController(BaseStorageServer):
         self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                             self.mount_check,
                                             logger=self.logger)
-        self.auto_create_account_prefix = \
-            conf.get('auto_create_account_prefix') or '.'
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
+        swift.common.db.QUERY_LOGGING = \
+            config_true_value(conf.get('db_query_logging', 'f'))
         self.fallocate_reserve, self.fallocate_is_percent = \
             config_fallocate_value(conf.get('fallocate_reserve', '1%'))
 
@@ -96,7 +127,7 @@ class AccountController(BaseStorageServer):
     @timing_stats()
     def DELETE(self, req):
         """Handle HTTP DELETE request."""
-        drive, part, account = split_and_validate_path(req, 3)
+        drive, part, account = get_account_name_and_placement(req)
         try:
             check_drive(self.root, drive, self.mount_check)
         except ValueError:
@@ -120,7 +151,7 @@ class AccountController(BaseStorageServer):
     @timing_stats()
     def PUT(self, req):
         """Handle HTTP PUT request."""
-        drive, part, account, container = split_and_validate_path(req, 3, 4)
+        drive, part, account, container = get_container_name_and_placement(req)
         try:
             check_drive(self.root, drive, self.mount_check)
         except ValueError:
@@ -145,8 +176,9 @@ class AccountController(BaseStorageServer):
                     broker.initialize(timestamp.internal)
                 except DatabaseAlreadyExists:
                     pass
-            if req.headers.get('x-account-override-deleted', 'no').lower() != \
-                    'yes' and broker.is_deleted():
+            if (req.headers.get('x-account-override-deleted', 'no').lower() !=
+                    'yes' and broker.is_deleted()) \
+                    or not os.path.exists(broker.db_file):
                 return HTTPNotFound(request=req)
             broker.put_container(container, req.headers['x-put-timestamp'],
                                  req.headers['x-delete-timestamp'],
@@ -185,7 +217,7 @@ class AccountController(BaseStorageServer):
     @timing_stats()
     def HEAD(self, req):
         """Handle HTTP HEAD request."""
-        drive, part, account = split_and_validate_path(req, 3)
+        drive, part, account = get_account_name_and_placement(req)
         out_content_type = listing_formats.get_listing_content_type(req)
         try:
             check_drive(self.root, drive, self.mount_check)
@@ -198,25 +230,18 @@ class AccountController(BaseStorageServer):
             return self._deleted_response(broker, req, HTTPNotFound)
         headers = get_response_headers(broker)
         headers['Content-Type'] = out_content_type
+        headers['Content-Length'] = 0
         return HTTPNoContent(request=req, headers=headers, charset='utf-8')
 
     @public
     @timing_stats()
     def GET(self, req):
         """Handle HTTP GET request."""
-        drive, part, account = split_and_validate_path(req, 3)
+        drive, part, account = get_account_name_and_placement(req)
         prefix = get_param(req, 'prefix')
         delimiter = get_param(req, 'delimiter')
-        limit = constraints.ACCOUNT_LISTING_LIMIT
-        given_limit = get_param(req, 'limit')
         reverse = config_true_value(get_param(req, 'reverse'))
-        if given_limit and given_limit.isdigit():
-            limit = int(given_limit)
-            if limit > constraints.ACCOUNT_LISTING_LIMIT:
-                return HTTPPreconditionFailed(
-                    request=req,
-                    body='Maximum limit is %d' %
-                    constraints.ACCOUNT_LISTING_LIMIT)
+        limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT)
         marker = get_param(req, 'marker', '')
         end_marker = get_param(req, 'end_marker')
         out_content_type = listing_formats.get_listing_content_type(req)
@@ -262,7 +287,7 @@ class AccountController(BaseStorageServer):
     @timing_stats()
     def POST(self, req):
         """Handle HTTP POST request."""
-        drive, part, account = split_and_validate_path(req, 3)
+        drive, part, account = get_account_name_and_placement(req)
         req_timestamp = valid_timestamp(req)
         try:
             check_drive(self.root, drive, self.mount_check)
@@ -280,8 +305,8 @@ class AccountController(BaseStorageServer):
         start_time = time.time()
         req = Request(env)
         self.logger.txn_id = req.headers.get('x-trans-id', None)
-        if not check_utf8(wsgi_to_str(req.path_info)):
-            res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
+        if not check_utf8(wsgi_to_str(req.path_info), internal=True):
+            res = HTTPPreconditionFailed(body='Invalid UTF8')
         else:
             try:
                 # disallow methods which are not publicly accessible
@@ -292,8 +317,8 @@ class AccountController(BaseStorageServer):
             except HTTPException as error_response:
                 res = error_response
             except (Exception, Timeout):
-                self.logger.exception(_('ERROR __call__ error with %(method)s'
-                                        ' %(path)s '),
+                self.logger.exception('ERROR __call__ error with %(method)s'
+                                      ' %(path)s ',
                                       {'method': req.method, 'path': req.path})
                 res = HTTPInternalServerError(body=traceback.format_exc())
         if self.log_requests:
@@ -317,3 +342,12 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return AccountController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'account-server', **options))
+
+
+if __name__ == '__main__':
+    main()

swift/account/utils.py

@@ -15,8 +15,7 @@
 
 import json
 
-import six
-
+from swift.common import constraints
 from swift.common.middleware import listing_formats
 from swift.common.swob import HTTPOk, HTTPNoContent, str_to_wsgi
 from swift.common.utils import Timestamp
@@ -71,25 +70,43 @@ def get_response_headers(broker):
 
 
 def account_listing_response(account, req, response_content_type, broker=None,
-                             limit='', marker='', end_marker='', prefix='',
-                             delimiter='', reverse=False):
+                             limit=constraints.ACCOUNT_LISTING_LIMIT,
+                             marker='', end_marker='', prefix='', delimiter='',
+                             reverse=False):
     if broker is None:
         broker = FakeAccountBroker()
 
     resp_headers = get_response_headers(broker)
 
     account_list = broker.list_containers_iter(limit, marker, end_marker,
-                                               prefix, delimiter, reverse)
+                                               prefix, delimiter, reverse,
+                                               req.allow_reserved_names)
     data = []
-    for (name, object_count, bytes_used, put_timestamp, is_subdir) \
+    for (name, object_count, bytes_used, put_timestamp,
+         storage_policy_index, is_subdir) \
             in account_list:
-        name_ = name.decode('utf8') if six.PY2 else name
         if is_subdir:
-            data.append({'subdir': name_})
+            data.append({'subdir': name})
         else:
-            data.append(
-                {'name': name_, 'count': object_count, 'bytes': bytes_used,
-                 'last_modified': Timestamp(put_timestamp).isoformat})
+            container = {
+                'name': name,
+                'count': object_count,
+                'bytes': bytes_used,
+                'last_modified': Timestamp(put_timestamp).isoformat}
+            # Add the container's storage policy to the response, unless
+            # storage_policy_index was not found in POLICIES, which means
+            # the storage policy is missing from the Swift configuration
+            # or otherwise could not be determined.
+            #
+            # The storage policy should always be returned when
+            # everything is configured correctly, but clients are
+            # expected to be able to handle this case regardless,
+            # if only to support older versions of swift.
+            if storage_policy_index in POLICIES:
+                container['storage_policy'] = (
+                    POLICIES[storage_policy_index].name
+                )
+            data.append(container)
     if response_content_type.endswith('/xml'):
         account_list = listing_formats.account_to_xml(data, account)
         ret = HTTPOk(body=account_list, request=req, headers=resp_headers)

swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,20 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import os
 import sys
-from hashlib import md5
 import getopt
 from itertools import chain
 
 import json
 from eventlet.greenpool import GreenPool
 from eventlet.event import Event
-from six.moves.urllib.parse import quote
+from urllib.parse import quote
 
 from swift.common.ring import Ring
 from swift.common.utils import split_path
+from swift.common.utils.base import md5
 from swift.common.bufferedhttp import http_connect
 
 
@@ -91,7 +90,7 @@ class Auditor(object):
                     conn = http_connect(node['ip'], node['port'],
                                         node['device'], part, 'GET', path, {})
                     resp = conn.getresponse()
-                    calc_hash = md5()
+                    calc_hash = md5(usedforsecurity=False)
                     chunk = True
                     while chunk:
                         chunk = resp.read(8192)
@@ -100,15 +99,15 @@ class Auditor(object):
                     if resp.status // 100 != 2:
                         self.object_not_found += 1
                         consistent = False
-                        print('  Bad status GETting object "%s" on %s/%s'
-                              % (path, node['ip'], node['device']))
+                        print('  Bad status %s GETting object "%s" on %s/%s'
+                              % (resp.status, path,
+                                 node['ip'], node['device']))
                         continue
                     if resp.getheader('ETag').strip('"') != calc_hash:
                         self.object_checksum_mismatch += 1
                         consistent = False
                         print('  MD5 does not match etag for "%s" on %s/%s'
                               % (path, node['ip'], node['device']))
-                    etags.append((resp.getheader('ETag'), node))
                 else:
                     conn = http_connect(node['ip'], node['port'],
                                         node['device'], part, 'HEAD',
@@ -117,9 +116,16 @@ class Auditor(object):
                     if resp.status // 100 != 2:
                         self.object_not_found += 1
                         consistent = False
-                        print('  Bad status HEADing object "%s" on %s/%s'
-                              % (path, node['ip'], node['device']))
+                        print('  Bad status %s HEADing object "%s" on %s/%s'
+                              % (resp.status, path,
+                                 node['ip'], node['device']))
                         continue
+
+                override_etag = resp.getheader(
+                    'X-Object-Sysmeta-Container-Update-Override-Etag')
+                if override_etag:
+                    etags.append((override_etag, node))
+                else:
                     etags.append((resp.getheader('ETag'), node))
             except Exception:
                 self.object_exceptions += 1
@@ -252,14 +258,14 @@ class Auditor(object):
                         self.account_not_found += 1
                         consistent = False
                         print("  Bad status GETting account '%s' "
-                              " from %ss:%ss" %
+                              " from %s:%s" %
                               (account, node['ip'], node['device']))
                         break
                     results = json.loads(resp.read())
                 except Exception:
                     self.account_exceptions += 1
                     consistent = False
-                    print("  Exception GETting account '%s' on %ss:%ss" %
+                    print("  Exception GETting account '%s' on %s:%s" %
                           (account, node['ip'], node['device']))
                     break
                 if node_id not in responses:
@@ -352,7 +358,7 @@ class Auditor(object):
             _print_stat("MD5 Mismatch", self.object_checksum_mismatch)
 
 
-if __name__ == '__main__':
+def main():
     try:
         optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
     except getopt.GetoptError as err:
@@ -377,3 +383,7 @@ if __name__ == '__main__':
         auditor.audit(*split_path(path, 1, 3, True))
     auditor.wait()
     auditor.print_stats()
+
+
+if __name__ == '__main__':
+    main()

swift-2.23.3.data/scripts/swift-config → swift/cli/config.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import optparse
 import os
 import sys
@@ -86,5 +85,6 @@ def main():
             print('# %s = %s' % (k, v))
         print()
 
+
 if __name__ == "__main__":
     sys.exit(main())

swift/cli/container_deleter.py

@@ -28,7 +28,6 @@ import argparse
 import io
 import itertools
 import json
-import six
 import time
 
 from swift.common.internal_client import InternalClient
@@ -50,17 +49,11 @@ def make_delete_jobs(account, container, objects, timestamp):
     :returns: list of dicts appropriate for an UPDATE request to an
               expiring-object queue
     '''
-    if six.PY2:
-        if isinstance(account, str):
-            account = account.decode('utf8')
-        if isinstance(container, str):
-            container = container.decode('utf8')
     return [
         {
             'name': build_task_obj(
                 timestamp, account, container,
-                obj.decode('utf8') if six.PY2 and isinstance(obj, str)
-                else obj),
+                obj, high_precision=True),
             'deleted': 0,
             'created_at': timestamp.internal,
             'etag': MD5_OF_EMPTY_STRING,
@@ -133,7 +126,7 @@ def mark_for_deletion(swift, account, container, marker, end_marker,
         return enqueue_deletes()
 
 
-def main():
+def main(args=None):
     parser = argparse.ArgumentParser(
         description=__doc__,
         formatter_class=argparse.RawTextHelpFormatter)
@@ -156,10 +149,11 @@ def main():
     parser.add_argument(
         '--timestamp', type=Timestamp, default=Timestamp.now(),
         help='delete all objects as of this time (default: now)')
-    args = parser.parse_args()
+    args = parser.parse_args(args)
 
     swift = InternalClient(
-        args.config, 'Swift Container Deleter', args.request_tries)
+        args.config, 'Swift Container Deleter', args.request_tries,
+        global_conf={'log_name': 'container-deleter-ic'})
     for deleted, marker in mark_for_deletion(
             swift, args.account, args.container,
             args.marker, args.end_marker, args.prefix, args.timestamp):

swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,6 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import print_function
 
 import io
 import traceback
@@ -23,9 +22,7 @@ from time import time
 
 from eventlet import GreenPool, patcher, sleep
 from eventlet.pools import Pool
-import six
-from six.moves import range
-from six.moves.configparser import ConfigParser
+from configparser import ConfigParser
 
 from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
@@ -53,7 +50,7 @@ def put_object(connpool, container, obj, report):
     global retries_done
     try:
         with connpool.item() as conn:
-            data = io.BytesIO(obj if six.PY2 else obj.encode('utf8'))
+            data = io.BytesIO(obj.encode('utf8'))
             conn.put_object(container, obj, data,
                             headers={'x-object-meta-dispersion': obj})
             retries_done += conn.attempts - 1
@@ -81,7 +78,7 @@ def report(success):
     stdout.flush()
 
 
-if __name__ == '__main__':
+def main():
     global begun, created, item_type, next_report, need_to_create, retries_done
     patcher.monkey_patch()
     try:
@@ -281,3 +278,7 @@ Usage: %%prog [options] [conf_file]
             print('\r\x1B[KTotal object coverage is now %.2f%%.' %
                   ((float(obj_coverage) / object_ring.partition_count * 100)))
         stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

swift/cli/dispersion_report.py

@@ -14,10 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import json
 from collections import defaultdict
-from six.moves.configparser import ConfigParser
+from configparser import ConfigParser
 from optparse import OptionParser
 from sys import exit, stdout, stderr
 from time import time
@@ -26,14 +25,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout
 from eventlet.pools import Pool
 
 from swift.common import direct_client
-try:
-    from swiftclient import get_auth
-except ImportError:
-    from swift.common.internal_client import get_auth
 from swift.common.internal_client import SimpleClient
 from swift.common.ring import Ring
 from swift.common.exceptions import ClientException
-from swift.common.utils import compute_eta, get_time_units, config_true_value
+from swift.common.utils import compute_eta, get_time_units, \
+    config_true_value, node_to_string
 from swift.common.storage_policy import POLICIES
 
 
@@ -93,7 +89,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
     def direct(container, part, nodes):
         found_count = 0
         for node in nodes:
-            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
+            error_log = get_error_log(node_to_string(node))
             try:
                 attempts, _junk = direct_client.retry(
                     direct_client.direct_head_container, node, part, account,
@@ -205,7 +201,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
     def direct(obj, part, nodes):
         found_count = 0
         for node in nodes:
-            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
+            error_log = get_error_log(node_to_string(node))
             try:
                 attempts, _junk = direct_client.retry(
                     direct_client.direct_head_object, node, part, account,
@@ -365,6 +361,11 @@ Usage: %%prog [options] [conf_file]
 
 
 def generate_report(conf, policy_name=None):
+    try:
+        # Delay importing so urllib3 will import monkey-patched modules
+        from swiftclient import get_auth
+    except ImportError:
+        from swift.common.internal_client import get_auth
     global json_output
     json_output = config_true_value(conf.get('dump_json', 'no'))
     if policy_name is None: