swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/bufferedhttp.py

@@ -27,28 +27,22 @@ BufferedHTTPResponse.
 """
 
 from swift.common import constraints
+import http.client
 import logging
 import time
 import socket
 
-import eventlet
-from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \
-    HTTPResponse, HTTPSConnection, _UNKNOWN
-from six.moves.urllib.parse import quote, parse_qsl, urlencode
-import six
+from eventlet.green.http.client import CONTINUE, HTTPConnection, \
+    HTTPResponse, HTTPSConnection, _UNKNOWN, ImproperConnectionState
+from urllib.parse import quote, parse_qsl, urlencode
 
-if six.PY2:
-    httplib = eventlet.import_patched('httplib')
-    from eventlet.green import httplib as green_httplib
-else:
-    httplib = eventlet.import_patched('http.client')
-    from eventlet.green.http import client as green_httplib
+from eventlet.green.http import client as green_http_client
 
 # Apparently http.server uses this to decide when/whether to send a 431.
 # Give it some slack, so the app is more likely to get the chance to reject
 # with a 400 instead.
-httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT * 1.6
-green_httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT * 1.6
+http.client._MAXHEADERS = constraints.MAX_HEADER_COUNT * 1.6
+green_http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT * 1.6
 
 
 class BufferedHTTPResponse(HTTPResponse):
@@ -56,21 +50,24 @@ class BufferedHTTPResponse(HTTPResponse):
 
     def __init__(self, sock, debuglevel=0, strict=0,
                  method=None):          # pragma: no cover
+        # sock should be an eventlet.greenio.GreenSocket
         self.sock = sock
-        # sock is an eventlet.greenio.GreenSocket
-        if six.PY2:
-            # sock.fd is a socket._socketobject
-            # sock.fd._sock is a _socket.socket object, which is what we want.
-            self._real_socket = sock.fd._sock
+        if sock is None:
+            # ...but it could be None if we close the connection as we're
+            # getting it wrapped up in a Response
+            self._real_socket = None
+            # No socket means no file-like -- set it to None like in
+            # HTTPResponse.close()
+            self.fp = None
         else:
             # sock.fd is a socket.socket, which should have a _real_close
             self._real_socket = sock.fd
-        self.fp = sock.makefile('rb')
+            self.fp = sock.makefile('rb')
         self.debuglevel = debuglevel
         self.strict = strict
         self._method = method
 
-        self.headers = self.msg = None
+        self._headers = self.msg = None
 
         # from the Status-Line of the response
         self.version = _UNKNOWN         # HTTP-Version
@@ -83,10 +80,17 @@ class BufferedHTTPResponse(HTTPResponse):
         self.will_close = _UNKNOWN      # conn will close at end of response
         self._readline_buffer = b''
 
-    if not six.PY2:
-        def begin(self):
-            HTTPResponse.begin(self)
-            header_payload = self.headers.get_payload()
+    @property
+    def headers(self):
+        return self._headers
+
+    @headers.setter
+    def headers(self, hdrs):
+        try:
+            header_payload = hdrs.get_payload()
+        except AttributeError:
+            pass
+        else:
             if isinstance(header_payload, list) and len(header_payload) == 1:
                 header_payload = header_payload[0].get_payload()
             if header_payload:
@@ -100,12 +104,19 @@ class BufferedHTTPResponse(HTTPResponse):
                         break
                     header, value = line.split(':', 1)
                     value = value.strip(' \t\n\r')
-                    self.headers.add_header(header, value)
+                    hdrs.add_header(header, value)
+                # Clear the payload now that all headers are present.
+                # Otherwise, we may double-up the headers parsed here
+                # if/when repeatedly setting the headers property.
+                hdrs.set_payload(None)
+        self._headers = hdrs
 
     def expect_response(self):
         if self.fp:
             self.fp.close()
             self.fp = None
+        if not self.sock:
+            raise ImproperConnectionState('Socket already closed')
         self.fp = self.sock.makefile('rb', 0)
         version, status, reason = self._read_status()
         if status != CONTINUE:
@@ -115,15 +126,7 @@ class BufferedHTTPResponse(HTTPResponse):
             self.status = status
             self.reason = reason.strip()
             self.version = 11
-            if six.PY2:
-                # Under py2, HTTPMessage.__init__ reads the headers
-                # which advances fp
-                self.msg = HTTPMessage(self.fp, 0)
-                # immediately kill msg.fp to make sure it isn't read again
-                self.msg.fp = None
-            else:
-                # py3 has a separate helper for it
-                self.headers = self.msg = httplib.parse_headers(self.fp)
+            self.headers = self.msg = http.client.parse_headers(self.fp)
 
     def read(self, amt=None):
         if not self._readline_buffer:
@@ -147,26 +150,6 @@ class BufferedHTTPResponse(HTTPResponse):
             self._readline_buffer = b''
             return buf + HTTPResponse.read(self, smaller_amt)
 
-    def readline(self, size=1024):
-        # You'd think Python's httplib would provide this, but it doesn't.
-        # It does, however, provide a comment in the HTTPResponse class:
-        #
-        #  # XXX It would be nice to have readline and __iter__ for this,
-        #  # too.
-        #
-        # Yes, it certainly would.
-        while (b'\n' not in self._readline_buffer
-               and len(self._readline_buffer) < size):
-            read_size = size - len(self._readline_buffer)
-            chunk = HTTPResponse.read(self, read_size)
-            if not chunk:
-                break
-            self._readline_buffer += chunk
-
-        line, newline, rest = self._readline_buffer.partition(b'\n')
-        self._readline_buffer = rest
-        return line + newline
-
     def nuke_from_orbit(self):
         """
         Terminate the socket with extreme prejudice.
@@ -176,14 +159,9 @@ class BufferedHTTPResponse(HTTPResponse):
         you care about has a reference to this socket.
         """
         if self._real_socket:
-            if six.PY2:
-                # this is idempotent; see sock_close in Modules/socketmodule.c
-                # in the Python source for details.
-                self._real_socket.close()
-            else:
-                # Hopefully this is equivalent?
-                # TODO: verify that this does everything ^^^^ does for py2
-                self._real_socket._real_close()
+            # Hopefully this is equivalent to py2's _real_socket.close()?
+            # TODO: verify that this does everything ^^^^ does for py2
+            self._real_socket._real_close()
         self._real_socket = None
         self.close()
 
@@ -258,13 +236,13 @@ def http_connect(ipaddr, port, device, partition, method, path,
     :param ssl: set True if SSL should be used (default: False)
     :returns: HTTPConnection object
     """
-    if isinstance(path, six.text_type):
+    if isinstance(path, str):
         path = path.encode("utf-8")
-    if isinstance(device, six.text_type):
+    if isinstance(device, str):
         device = device.encode("utf-8")
-    if isinstance(partition, six.text_type):
+    if isinstance(partition, str):
         partition = partition.encode('utf-8')
-    elif isinstance(partition, six.integer_types):
+    elif isinstance(partition, int):
         partition = str(partition).encode('ascii')
     path = quote(b'/' + device + b'/' + partition + path)
     return http_connect_raw(
@@ -295,14 +273,10 @@ def http_connect_raw(ipaddr, port, method, path, headers=None,
         conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port))
     if query_string:
         # Round trip to ensure proper quoting
-        if six.PY2:
-            query_string = urlencode(parse_qsl(
-                query_string, keep_blank_values=True))
-        else:
-            query_string = urlencode(
-                parse_qsl(query_string, keep_blank_values=True,
-                          encoding='latin1'),
-                encoding='latin1')
+        query_string = urlencode(
+            parse_qsl(query_string, keep_blank_values=True,
+                      encoding='latin1'),
+            encoding='latin1')
         path += '?' + query_string
     conn.path = path
     conn.putrequest(method, path, skip_host=(headers and 'Host' in headers))

swift/common/constraints.py

@@ -17,9 +17,8 @@ import functools
 import os
 from os.path import isdir  # tighter scoped import for mocking
 
-import six
-from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
-from six.moves import urllib
+from configparser import ConfigParser, NoSectionError, NoOptionError
+import urllib
 
 from swift.common import utils, exceptions
 from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
@@ -39,6 +38,7 @@ MAX_ACCOUNT_NAME_LENGTH = 256
 MAX_CONTAINER_NAME_LENGTH = 256
 VALID_API_VERSIONS = ["v1", "v1.0"]
 EXTRA_HEADER_COUNT = 0
+AUTO_CREATE_ACCOUNT_PREFIX = '.'
 
 # If adding an entry to DEFAULT_CONSTRAINTS, note that
 # these constraints are automatically published by the
@@ -58,6 +58,7 @@ DEFAULT_CONSTRAINTS = {
     'max_container_name_length': MAX_CONTAINER_NAME_LENGTH,
     'valid_api_versions': VALID_API_VERSIONS,
     'extra_header_count': EXTRA_HEADER_COUNT,
+    'auto_create_account_prefix': AUTO_CREATE_ACCOUNT_PREFIX,
 }
 
 SWIFT_CONSTRAINTS_LOADED = False
@@ -76,7 +77,7 @@ def reload_constraints():
     constraints_conf = ConfigParser()
     if constraints_conf.read(utils.SWIFT_CONF_FILE):
         SWIFT_CONSTRAINTS_LOADED = True
-        for name in DEFAULT_CONSTRAINTS:
+        for name, default in DEFAULT_CONSTRAINTS.items():
             try:
                 value = constraints_conf.get('swift-constraints', name)
             except NoOptionError:
@@ -85,9 +86,12 @@ def reload_constraints():
                 # We are never going to find the section for another option
                 break
             else:
-                try:
-                    value = int(value)
-                except ValueError:
+                if isinstance(default, int):
+                    value = int(value)  # Go ahead and let it error
+                elif isinstance(default, str):
+                    pass  # No translation needed, I guess
+                else:
+                    # Hope we want a list!
                     value = utils.list_from_csv(value)
                 OVERRIDE_CONSTRAINTS[name] = value
     for name, default in DEFAULT_CONSTRAINTS.items():
@@ -125,7 +129,7 @@ def check_metadata(req, target_type):
     meta_count = 0
     meta_size = 0
     for key, value in req.headers.items():
-        if (isinstance(value, six.string_types)
+        if (isinstance(value, str)
            and len(value) > MAX_HEADER_SIZE):
 
             return HTTPBadRequest(body=b'Header value too long: %s' %
@@ -346,19 +350,20 @@ def check_delete_headers(request):
     return request
 
 
-def check_utf8(string):
+def check_utf8(string, internal=False):
     """
     Validate if a string is valid UTF-8 str or unicode and that it
-    does not contain any null character.
+    does not contain any reserved characters.
 
     :param string: string to be validated
+    :param internal: boolean, allows reserved characters if True
     :returns: True if the string is valid utf-8 str or unicode and
               contains no null characters, False otherwise
     """
     if not string:
         return False
     try:
-        if isinstance(string, six.text_type):
+        if isinstance(string, str):
             encoded = string.encode('utf-8')
             decoded = string
         else:
@@ -382,7 +387,9 @@ def check_utf8(string):
         if any(0xD800 <= ord(codepoint) <= 0xDFFF
                for codepoint in decoded):
             return False
-        return b'\x00' not in encoded
+        if b'\x00' != utils.RESERVED_BYTE and b'\x00' in encoded:
+            return False
+        return True if internal else utils.RESERVED_BYTE not in encoded
     # If string is unicode, decode() will raise UnicodeEncodeError
     # So, we should catch both UnicodeDecodeError & UnicodeEncodeError
     except UnicodeError:
@@ -404,15 +411,13 @@ def check_name_format(req, name, target_type):
         raise HTTPPreconditionFailed(
             request=req,
             body='%s name cannot be empty' % target_type)
-    if six.PY2:
-        if isinstance(name, six.text_type):
-            name = name.encode('utf-8')
     if '/' in name:
         raise HTTPPreconditionFailed(
             request=req,
             body='%s name cannot contain slashes' % target_type)
     return name
 
+
 check_account_format = functools.partial(check_name_format,
                                          target_type='Account')
 check_container_format = functools.partial(check_name_format,

swift/common/container_sync_realms.py

@@ -19,10 +19,8 @@ import hmac
 import os
 import time
 
-import six
-from six.moves import configparser
+import configparser
 
-from swift import gettext_ as _
 from swift.common.utils import get_valid_utf8_str
 
 
@@ -58,7 +56,7 @@ class ContainerSyncRealms(object):
                     log_func = self.logger.debug
                 else:
                     log_func = self.logger.error
-                log_func(_('Could not load %(conf)r: %(error)s') % {
+                log_func('Could not load %(conf)r: %(error)s', {
                          'conf': self.conf_path, 'error': err})
             else:
                 if mtime != self.conf_path_mtime:
@@ -68,11 +66,11 @@ class ContainerSyncRealms(object):
                         conf.read(self.conf_path)
                     except configparser.ParsingError as err:
                         self.logger.error(
-                            _('Could not load %(conf)r: %(error)s')
-                            % {'conf': self.conf_path, 'error': err})
+                            'Could not load %(conf)r: %(error)s',
+                            {'conf': self.conf_path, 'error': err})
                     else:
                         try:
-                            self.mtime_check_interval = conf.getint(
+                            self.mtime_check_interval = conf.getfloat(
                                 'DEFAULT', 'mtime_check_interval')
                             self.next_mtime_check = \
                                 now + self.mtime_check_interval
@@ -82,9 +80,9 @@ class ContainerSyncRealms(object):
                                 now + self.mtime_check_interval
                         except (configparser.ParsingError, ValueError) as err:
                             self.logger.error(
-                                _('Error in %(conf)r with '
-                                  'mtime_check_interval: %(error)s')
-                                % {'conf': self.conf_path, 'error': err})
+                                'Error in %(conf)r with '
+                                'mtime_check_interval: %(error)s',
+                                {'conf': self.conf_path, 'error': err})
                         realms = {}
                         for section in conf.sections():
                             realm = {}
@@ -159,7 +157,7 @@ class ContainerSyncRealms(object):
         user_key = get_valid_utf8_str(user_key)
         # XXX We don't know what is the best here yet; wait for container
         # sync to be tested.
-        if isinstance(path, six.text_type):
+        if isinstance(path, str):
             path = path.encode('utf-8')
         return hmac.new(
             realm_key,

swift/common/daemon.py

@@ -20,8 +20,8 @@ import time
 import signal
 from re import sub
 
+import eventlet
 import eventlet.debug
-from eventlet.hubs import use_hub
 
 from swift.common import utils
 
@@ -45,6 +45,7 @@ class Daemon(object):
     multiple daemonized workers, they simply provide the behavior of the daemon
     and context specific knowledge about how workers should be started.
     """
+    WORKERS_HEALTHCHECK_INTERVAL = 5.0
 
     def __init__(self, conf):
         self.conf = conf
@@ -132,17 +133,19 @@ class DaemonStrategy(object):
     def setup(self, **kwargs):
         utils.validate_configuration()
         utils.drop_privileges(self.daemon.conf.get('user', 'swift'))
+        utils.clean_up_daemon_hygiene()
         utils.capture_stdio(self.logger, **kwargs)
 
         def kill_children(*args):
             self.running = False
-            self.logger.info('SIGTERM received')
+            self.logger.notice('SIGTERM received (%s)', os.getpid())
             signal.signal(signal.SIGTERM, signal.SIG_IGN)
             os.killpg(0, signal.SIGTERM)
             os._exit(0)
 
         signal.signal(signal.SIGTERM, kill_children)
         self.running = True
+        utils.systemd_notify(self.logger)
 
     def _run_inline(self, once=False, **kwargs):
         """Run the daemon"""
@@ -156,7 +159,7 @@ class DaemonStrategy(object):
         except KeyboardInterrupt:
             self.logger.notice('User quit')
         finally:
-            self.cleanup()
+            self.cleanup(stopping=True)
         self.running = False
 
     def _fork(self, once, **kwargs):
@@ -164,6 +167,8 @@ class DaemonStrategy(object):
         if pid == 0:
             signal.signal(signal.SIGHUP, signal.SIG_DFL)
             signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            # only MAINPID should be sending notifications
+            os.environ.pop('NOTIFY_SOCKET', None)
 
             self.daemon.run(once, **kwargs)
 
@@ -238,11 +243,19 @@ class DaemonStrategy(object):
                 if not self.spawned_pids():
                     self.logger.notice('Finished %s', os.getpid())
                     break
-            time.sleep(0.1)
+            time.sleep(self.daemon.WORKERS_HEALTHCHECK_INTERVAL)
         self.daemon.post_multiprocess_run()
         return 0
 
-    def cleanup(self):
+    def cleanup(self, stopping=False):
+        """
+        Cleanup worker processes
+
+        :param stopping: if set, tell systemd we're stopping
+        """
+
+        if stopping:
+            utils.systemd_notify(self.logger, "STOPPING=1")
         for p in self.spawned_pids():
             try:
                 os.kill(p, signal.SIGTERM)
@@ -267,7 +280,7 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
     """
     # very often the config section_name is based on the class name
     # the None singleton will be passed through to readconf as is
-    if section_name is '':
+    if section_name == '':
         section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
                            klass.__name__).lower()
     try:
@@ -278,7 +291,9 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
         # and results in an exit code of 1.
         sys.exit(e)
 
-    use_hub(utils.get_hub())
+    # patch eventlet/logging early
+    utils.monkey_patch()
+    eventlet.hubs.use_hub(utils.get_hub())
 
     # once on command line (i.e. daemonize=false) will over-ride config
     once = once or not utils.config_true_value(conf.get('daemonize', 'true'))
@@ -312,7 +327,9 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
 
     logger.notice('Starting %s', os.getpid())
     try:
-        DaemonStrategy(klass(conf), logger).run(once=once, **kwargs)
+        d = klass(conf)
+        DaemonStrategy(d, logger).run(once=once, **kwargs)
     except KeyboardInterrupt:
         logger.info('User quit')
     logger.notice('Exited %s', os.getpid())
+    return d