swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/obj/ssync_receiver.py

@@ -16,7 +16,8 @@
 
 import eventlet.greenio
 import eventlet.wsgi
-from six.moves import urllib
+from eventlet import sleep
+import urllib
 
 from swift.common import exceptions
 from swift.common import http
@@ -26,11 +27,16 @@ from swift.common import request_helpers
 from swift.common.utils import Timestamp
 
 
+class SsyncClientDisconnected(Exception):
+    pass
+
+
 def decode_missing(line):
     """
     Parse a string of the form generated by
     :py:func:`~swift.obj.ssync_sender.encode_missing` and return a dict
-    with keys ``object_hash``, ``ts_data``, ``ts_meta``, ``ts_ctype``.
+    with keys ``object_hash``, ``ts_data``, ``ts_meta``, ``ts_ctype``,
+    ``durable``.
 
     The encoder for this line is
     :py:func:`~swift.obj.ssync_sender.encode_missing`
@@ -39,17 +45,28 @@ def decode_missing(line):
     parts = line.decode('ascii').split()
     result['object_hash'] = urllib.parse.unquote(parts[0])
     t_data = urllib.parse.unquote(parts[1])
-    result['ts_data'] = Timestamp(t_data)
-    result['ts_meta'] = result['ts_ctype'] = result['ts_data']
+    result['ts_data'] = ts_data = Timestamp(t_data)
+    result['ts_meta'] = result['ts_ctype'] = ts_data
+    result['durable'] = True  # default to True in case this key isn't sent
     if len(parts) > 2:
         # allow for a comma separated list of k:v pairs to future-proof
         subparts = urllib.parse.unquote(parts[2]).split(',')
         for item in [subpart for subpart in subparts if ':' in subpart]:
             k, v = item.split(':')
             if k == 'm':
-                result['ts_meta'] = Timestamp(t_data, delta=int(v, 16))
+                v, _, o = v.partition('__')
+                # ignore ts_data offset when calculating ts_meta
+                result['ts_meta'] = Timestamp(ts_data.normal,
+                                              delta=int(v, 16),
+                                              offset=int(o or '0', 16))
             elif k == 't':
-                result['ts_ctype'] = Timestamp(t_data, delta=int(v, 16))
+                v, _, o = v.partition('__')
+                # ignore ts_data offset when calculating ts_ctype
+                result['ts_ctype'] = Timestamp(Timestamp(ts_data).normal,
+                                               delta=int(v, 16),
+                                               offset=int(o or '0', 16))
+            elif k == 'durable':
+                result['durable'] = utils.config_true_value(v)
     return result
 
 
@@ -107,17 +124,17 @@ class Receiver(object):
 
     The general process inside an SSYNC request is:
 
-        1. Initialize the request: Basic request validation, mount check,
-           acquire semaphore lock, etc..
+    1. Initialize the request: Basic request validation, mount check,
+       acquire semaphore lock, etc..
 
-        2. Missing check: Sender sends the hashes and timestamps of
-           the object information it can send, receiver sends back
-           the hashes it wants (doesn't have or has an older
-           timestamp).
+    2. Missing check: Sender sends the hashes and timestamps of
+       the object information it can send, receiver sends back
+       the hashes it wants (doesn't have or has an older
+       timestamp).
 
-        3. Updates: Sender sends the object information requested.
+    3. Updates: Sender sends the object information requested.
 
-        4. Close down: Release semaphore lock, etc.
+    4. Close down: Release semaphore lock, etc.
     """
 
     def __init__(self, app, request):
@@ -141,6 +158,7 @@ class Receiver(object):
         """
         # The general theme for functions __call__ calls is that they should
         # raise exceptions.MessageTimeout for client timeouts (logged locally),
+        # exceptions.ChunkReadError for client disconnects (logged locally),
         # swob.HTTPException classes for exceptions to return to the caller but
         # not log locally (unmounted, for example), and any other Exceptions
         # will be logged with a full stack trace.
@@ -172,7 +190,10 @@ class Receiver(object):
                 finally:
                     if self.app.replication_semaphore:
                         self.app.replication_semaphore.release()
-            except exceptions.ReplicationLockTimeout as err:
+            except SsyncClientDisconnected:
+                self.app.logger.error('ssync client disconnected')
+                self.disconnect = True
+            except exceptions.LockTimeout as err:
                 self.app.logger.debug(
                     '%s/%s/%s SSYNC LOCK TIMEOUT: %s' % (
                         self.request.remote_addr, self.device, self.partition,
@@ -184,6 +205,11 @@ class Receiver(object):
                         self.request.remote_addr, self.device, self.partition,
                         err))
                 yield (':ERROR: %d %r\n' % (408, str(err))).encode('utf8')
+            except exceptions.ChunkReadError as err:
+                self.app.logger.error(
+                    '%s/%s/%s read failed in ssync.Receiver: %s' % (
+                        self.request.remote_addr, self.device, self.partition,
+                        err))
             except swob.HTTPException as err:
                 body = b''.join(err({}, lambda *args: None))
                 yield (':ERROR: %d %r\n' % (
@@ -236,6 +262,17 @@ class Receiver(object):
             raise swob.HTTPInsufficientStorage(drive=self.device)
         self.fp = self.request.environ['wsgi.input']
 
+    def _readline(self, context):
+        # try to read a line from the wsgi input; annotate any timeout or read
+        # errors with a description of the calling context
+        with exceptions.MessageTimeout(
+                self.app.client_timeout, context):
+            try:
+                line = self.fp.readline(self.app.network_chunk_size)
+            except (eventlet.wsgi.ChunkReadError, IOError) as err:
+                raise exceptions.ChunkReadError('%s: %s' % (context, err))
+            return line
+
     def _check_local(self, remote, make_durable=True):
         """
         Parse local diskfile and return results of current
@@ -254,6 +291,7 @@ class Receiver(object):
         except exceptions.DiskFileDeleted as err:
             result = {'ts_data': err.timestamp}
         except exceptions.DiskFileError:
+            # e.g. a non-durable EC frag
             result = {}
         else:
             result = {
@@ -261,25 +299,35 @@ class Receiver(object):
                 'ts_meta': df.timestamp,
                 'ts_ctype': df.content_type_timestamp,
             }
-        if (make_durable and df.fragments and
-            remote['ts_data'] in df.fragments and
-            self.frag_index in df.fragments[remote['ts_data']] and
-            (df.durable_timestamp is None or
-             df.durable_timestamp < remote['ts_data'])):
-            # We have the frag, just missing durable state, so make the frag
-            # durable now. Try this just once to avoid looping if it fails.
-            try:
-                with df.create() as writer:
-                    writer.commit(remote['ts_data'])
-                return self._check_local(remote, make_durable=False)
-            except Exception:
-                # if commit fails then log exception and fall back to wanting
-                # a full update
-                self.app.logger.exception(
-                    '%s/%s/%s EXCEPTION in ssync.Receiver while '
-                    'attempting commit of %s'
-                    % (self.request.remote_addr, self.device, self.partition,
-                       df._datadir))
+        if ((df.durable_timestamp is None or
+             df.durable_timestamp < remote['ts_data']) and
+                df.fragments and
+                remote['ts_data'] in df.fragments and
+                self.frag_index in df.fragments[remote['ts_data']]):
+            # The remote is offering a fragment that we already have but is
+            # *newer* than anything *durable* that we have
+            if remote['durable']:
+                # We have the frag, just missing durable state, so make the
+                # frag durable now. Try this just once to avoid looping if
+                # it fails.
+                if make_durable:
+                    try:
+                        with df.create() as writer:
+                            writer.commit(remote['ts_data'])
+                        return self._check_local(remote, make_durable=False)
+                    except Exception:
+                        # if commit fails then log exception and fall back to
+                        # wanting a full update
+                        self.app.logger.exception(
+                            '%s/%s/%s EXCEPTION in ssync.Receiver while '
+                            'attempting commit of %s'
+                            % (self.request.remote_addr, self.device,
+                               self.partition, df._datadir))
+            else:
+                # We have the non-durable frag that is on offer, but our
+                # ts_data may currently be set to an older durable frag, so
+                # bump our ts_data to prevent the remote frag being wanted.
+                result['ts_data'] = remote['ts_data']
         return result
 
     def _check_missing(self, line):
@@ -306,50 +354,54 @@ class Receiver(object):
 
         The process is generally:
 
-            1. Sender sends `:MISSING_CHECK: START` and begins
-               sending `hash timestamp` lines.
+        1. Sender sends ``:MISSING_CHECK: START`` and begins
+           sending `hash timestamp` lines.
 
-            2. Receiver gets `:MISSING_CHECK: START` and begins
-               reading the `hash timestamp` lines, collecting the
-               hashes of those it desires.
+        2. Receiver gets ``:MISSING_CHECK: START`` and begins
+           reading the `hash timestamp` lines, collecting the
+           hashes of those it desires.
 
-            3. Sender sends `:MISSING_CHECK: END`.
+        3. Sender sends ``:MISSING_CHECK: END``.
 
-            4. Receiver gets `:MISSING_CHECK: END`, responds with
-               `:MISSING_CHECK: START`, followed by the list of
-               <wanted_hash> specifiers it collected as being wanted
-               (one per line), `:MISSING_CHECK: END`, and flushes any
-               buffers.
+        4. Receiver gets ``:MISSING_CHECK: END``, responds with
+           ``:MISSING_CHECK: START``, followed by the list of
+           <wanted_hash> specifiers it collected as being wanted
+           (one per line), ``:MISSING_CHECK: END``, and flushes any
+           buffers.
 
-               Each <wanted_hash> specifier has the form <hash>[ <parts>] where
-               <parts> is a string containing characters 'd' and/or 'm'
-               indicating that only data or meta part of object respectively is
-               required to be sync'd.
+           Each <wanted_hash> specifier has the form <hash>[ <parts>] where
+           <parts> is a string containing characters 'd' and/or 'm'
+           indicating that only data or meta part of object respectively is
+           required to be sync'd.
 
-            5. Sender gets `:MISSING_CHECK: START` and reads the list
-               of hashes desired by the receiver until reading
-               `:MISSING_CHECK: END`.
+        5. Sender gets ``:MISSING_CHECK: START`` and reads the list
+           of hashes desired by the receiver until reading
+           ``:MISSING_CHECK: END``.
 
         The collection and then response is so the sender doesn't
         have to read while it writes to ensure network buffers don't
         fill up and block everything.
         """
-        with exceptions.MessageTimeout(
-                self.app.client_timeout, 'missing_check start'):
-            line = self.fp.readline(self.app.network_chunk_size)
+        line = self._readline('missing_check start')
+        if not line:
+            # Guess they hung up
+            raise SsyncClientDisconnected
         if line.strip() != b':MISSING_CHECK: START':
             raise Exception(
-                'Looking for :MISSING_CHECK: START got %r' % line[:1024])
+                'Looking for :MISSING_CHECK: START got %r'
+                % utils.cap_length(line, 1024))
         object_hashes = []
+        nlines = 0
         while True:
-            with exceptions.MessageTimeout(
-                    self.app.client_timeout, 'missing_check line'):
-                line = self.fp.readline(self.app.network_chunk_size)
+            line = self._readline('missing_check line')
             if not line or line.strip() == b':MISSING_CHECK: END':
                 break
             want = self._check_missing(line)
             if want:
                 object_hashes.append(want)
+            if nlines % 5 == 0:
+                sleep()  # Gives a chance for other greenthreads to run
+            nlines += 1
         yield b':MISSING_CHECK: START\r\n'
         if object_hashes:
             yield b'\r\n'.join(hsh.encode('ascii') for hsh in object_hashes)
@@ -370,18 +422,18 @@ class Receiver(object):
 
         The process is generally:
 
-            1. Sender sends `:UPDATES: START` and begins sending the
-               PUT and DELETE subrequests.
+        1. Sender sends ``:UPDATES: START`` and begins sending the
+           PUT and DELETE subrequests.
 
-            2. Receiver gets `:UPDATES: START` and begins routing the
-               subrequests to the object server.
+        2. Receiver gets ``:UPDATES: START`` and begins routing the
+           subrequests to the object server.
 
-            3. Sender sends `:UPDATES: END`.
+        3. Sender sends ``:UPDATES: END``.
 
-            4. Receiver gets `:UPDATES: END` and sends `:UPDATES:
-               START` and `:UPDATES: END` (assuming no errors).
+        4. Receiver gets ``:UPDATES: END`` and sends ``:UPDATES:
+           START`` and ``:UPDATES: END`` (assuming no errors).
 
-            5. Sender gets `:UPDATES: START` and `:UPDATES: END`.
+        5. Sender gets ``:UPDATES: START`` and ``:UPDATES: END``.
 
         If too many subrequests fail, as configured by
         replication_failure_threshold and replication_failure_ratio,
@@ -394,17 +446,18 @@ class Receiver(object):
         success. This is so the sender knows if it can remove an out
         of place partition, for example.
         """
-        with exceptions.MessageTimeout(
-                self.app.client_timeout, 'updates start'):
-            line = self.fp.readline(self.app.network_chunk_size)
+        line = self._readline('updates start')
+        if not line:
+            # Guess they hung up waiting for us to process the missing check
+            raise SsyncClientDisconnected
         if line.strip() != b':UPDATES: START':
-            raise Exception('Looking for :UPDATES: START got %r' % line[:1024])
+            raise Exception('Looking for :UPDATES: START got %r'
+                            % utils.cap_length(line, 1024))
         successes = 0
         failures = 0
+        updates = 0
         while True:
-            with exceptions.MessageTimeout(
-                    self.app.client_timeout, 'updates line'):
-                line = self.fp.readline(self.app.network_chunk_size)
+            line = self._readline('updates line')
             if not line or line.strip() == b':UPDATES: END':
                 break
             # Read first line METHOD PATH of subrequest.
@@ -416,22 +469,26 @@ class Receiver(object):
             content_length = None
             replication_headers = []
             while True:
-                with exceptions.MessageTimeout(self.app.client_timeout):
-                    line = self.fp.readline(self.app.network_chunk_size)
+                line = self._readline('updates line')
                 if not line:
                     raise Exception(
                         'Got no headers for %s %s' % (method, path))
                 line = line.strip()
                 if not line:
                     break
-                header, value = swob.bytes_to_wsgi(line).split(':', 1)
-                header = header.strip().lower()
-                value = value.strip()
+                header, value = line.split(b':', 1)
+                header = swob.bytes_to_wsgi(header.strip().lower())
+                value = swob.bytes_to_wsgi(value.strip())
                 subreq.headers[header] = value
-                if header != 'etag':
-                    # make sure ssync doesn't cause 'Etag' to be added to
-                    # obj metadata in addition to 'ETag' which object server
-                    # sets (note capitalization)
+                if header not in ('etag', 'x-backend-no-commit'):
+                    # we'll use X-Backend-Replication-Headers to force the
+                    # object server to write all sync'd metadata, but with some
+                    # exceptions:
+                    #  - make sure ssync doesn't cause 'Etag' to be added to
+                    #    obj metadata in addition to 'ETag' which object server
+                    #    sets (note capitalization)
+                    #  - filter out x-backend-no-commit which ssync sender may
+                    #    have added to the subrequest
                     replication_headers.append(header)
                 if header == 'content-length':
                     content_length = int(value)
@@ -478,8 +535,8 @@ class Receiver(object):
                 successes += 1
             else:
                 self.app.logger.warning(
-                    'ssync subrequest failed with %s: %s %s' %
-                    (resp.status_int, method, subreq.path))
+                    'ssync subrequest failed with %s: %s %s (%s)' %
+                    (resp.status_int, method, subreq.path, resp.body))
                 failures += 1
             if failures >= self.app.replication_failure_threshold and (
                     not successes or
@@ -493,6 +550,9 @@ class Receiver(object):
             # subreq.
             for junk in subreq.environ['wsgi.input']:
                 pass
+            if updates % 5 == 0:
+                sleep()  # Gives a chance for other greenthreads to run
+            updates += 1
         if failures:
             raise swob.HTTPInternalServerError(
                 'ERROR: With :UPDATES: %d failures to %d successes' %