swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/container/sync.py

@@ -17,84 +17,59 @@ import collections
 import errno
 import os
 import uuid
-from swift import gettext_ as _
 from time import ctime, time
 from random import choice, random
 from struct import unpack_from
 
 from eventlet import sleep, Timeout
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 import swift.common.db
 from swift.common.db import DatabaseConnectionError
 from swift.container.backend import ContainerBroker
 from swift.container.sync_store import ContainerSyncStore
 from swift.common.container_sync_realms import ContainerSyncRealms
+from swift.common.daemon import run_daemon
 from swift.common.internal_client import (
     delete_object, put_object, head_object,
     InternalClient, UnexpectedResponse)
 from swift.common.exceptions import ClientException
 from swift.common.ring import Ring
 from swift.common.ring.utils import is_local_device
+from swift.common.swob import normalize_etag
 from swift.common.utils import (
     clean_content_type, config_true_value,
     FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
-    whataremyips, Timestamp, decode_timestamps)
+    whataremyips, Timestamp, decode_timestamps, parse_options)
 from swift.common.daemon import Daemon
-from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
+from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
 from swift.common.wsgi import ConfigString
+from swift.common.middleware.versioned_writes.object_versioning import (
+    SYSMETA_VERSIONS_CONT, SYSMETA_VERSIONS_SYMLINK)
 
 
 # The default internal client config body is to support upgrades without
 # requiring deployment of the new /etc/swift/internal-client.conf
 ic_conf_body = """
 [DEFAULT]
-# swift_dir = /etc/swift
-# user = swift
-# You can specify default log routing here if you want:
-# log_name = swift
-# log_facility = LOG_LOCAL0
-# log_level = INFO
-# log_address = /dev/log
-#
-# comma separated list of functions to call to setup custom log handlers.
-# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
-# adapted_logger
-# log_custom_handlers =
-#
-# If set, log_udp_host will override log_address
-# log_udp_host =
-# log_udp_port = 514
-#
-# You can enable StatsD logging here:
-# log_statsd_host =
-# log_statsd_port = 8125
-# log_statsd_default_sample_rate = 1.0
-# log_statsd_sample_rate_factor = 1.0
-# log_statsd_metric_prefix =
-
 [pipeline:main]
 pipeline = catch_errors proxy-logging cache symlink proxy-server
 
 [app:proxy-server]
 use = egg:swift#proxy
 account_autocreate = true
-# See proxy-server.conf-sample for options
 
 [filter:symlink]
 use = egg:swift#symlink
-# See proxy-server.conf-sample for options
 
 [filter:cache]
 use = egg:swift#memcache
-# See proxy-server.conf-sample for options
 
 [filter:proxy-logging]
 use = egg:swift#proxy_logging
 
 [filter:catch_errors]
 use = egg:swift#catch_errors
-# See proxy-server.conf-sample for options
 """.lstrip()
 
 
@@ -122,28 +97,28 @@ class ContainerSync(Daemon):
     An example may help. Assume replica count is 3 and perfectly matching
     ROWIDs starting at 1.
 
-        First sync run, database has 6 rows:
-
-            * SyncPoint1 starts as -1.
-            * SyncPoint2 starts as -1.
-            * No rows between points, so no "all updates" rows.
-            * Six rows newer than SyncPoint1, so a third of the rows are sent
-              by node 1, another third by node 2, remaining third by node 3.
-            * SyncPoint1 is set as 6 (the newest ROWID known).
-            * SyncPoint2 is left as -1 since no "all updates" rows were synced.
-
-        Next sync run, database has 12 rows:
-
-            * SyncPoint1 starts as 6.
-            * SyncPoint2 starts as -1.
-            * The rows between -1 and 6 all trigger updates (most of which
-              should short-circuit on the remote end as having already been
-              done).
-            * Six more rows newer than SyncPoint1, so a third of the rows are
-              sent by node 1, another third by node 2, remaining third by node
-              3.
-            * SyncPoint1 is set as 12 (the newest ROWID known).
-            * SyncPoint2 is set as 6 (the newest "all updates" ROWID).
+       First sync run, database has 6 rows:
+
+       * SyncPoint1 starts as -1.
+       * SyncPoint2 starts as -1.
+       * No rows between points, so no "all updates" rows.
+       * Six rows newer than SyncPoint1, so a third of the rows are sent
+         by node 1, another third by node 2, remaining third by node 3.
+       * SyncPoint1 is set as 6 (the newest ROWID known).
+       * SyncPoint2 is left as -1 since no "all updates" rows were synced.
+
+       Next sync run, database has 12 rows:
+
+       * SyncPoint1 starts as 6.
+       * SyncPoint2 starts as -1.
+       * The rows between -1 and 6 all trigger updates (most of which
+         should short-circuit on the remote end as having already been
+         done).
+       * Six more rows newer than SyncPoint1, so a third of the rows are
+         sent by node 1, another third by node 2, remaining third by node
+         3.
+       * SyncPoint1 is set as 12 (the newest ROWID known).
+       * SyncPoint2 is set as 6 (the newest "all updates" ROWID).
 
     In this way, under normal circumstances each node sends its share of
     updates each run and just sends a batch of older updates to ensure nothing
@@ -154,13 +129,14 @@ class ContainerSync(Daemon):
     :param container_ring: If None, the <swift_dir>/container.ring.gz will be
                            loaded. This is overridden by unit tests.
     """
+    log_route = 'container-sync'
 
     def __init__(self, conf, container_ring=None, logger=None):
         #: The dict of configuration values from the [container-sync] section
         #: of the container-server.conf.
         self.conf = conf
         #: Logger to use for container-sync log lines.
-        self.logger = logger or get_logger(conf, log_route='container-sync')
+        self.logger = logger or get_logger(conf, log_route=self.log_route)
         #: Path to the local device mount points.
         self.devices = conf.get('devices', '/srv/node')
         #: Indicates whether mount points should be verified as actual mount
@@ -168,7 +144,7 @@ class ContainerSync(Daemon):
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
         #: Minimum time between full scans. This is to keep the daemon from
         #: running wild on near empty systems.
-        self.interval = int(conf.get('interval', 300))
+        self.interval = float(conf.get('interval', 300))
         #: Maximum amount of time to spend syncing a container before moving on
         #: to the next one. If a container sync hasn't finished in this time,
         #: it'll just be resumed next scan.
@@ -230,22 +206,25 @@ class ContainerSync(Daemon):
         internal_client_conf_path = conf.get('internal_client_conf_path')
         if not internal_client_conf_path:
             self.logger.warning(
-                _('Configuration option internal_client_conf_path not '
-                  'defined. Using default configuration, See '
-                  'internal-client.conf-sample for options'))
+                'Configuration option internal_client_conf_path not '
+                'defined. Using default configuration, See '
+                'internal-client.conf-sample for options')
             internal_client_conf = ConfigString(ic_conf_body)
         else:
             internal_client_conf = internal_client_conf_path
         try:
             self.swift = InternalClient(
-                internal_client_conf, 'Swift Container Sync', request_tries)
+                internal_client_conf, 'Swift Container Sync', request_tries,
+                use_replication_network=True,
+                global_conf={'log_name': '%s-ic' % conf.get(
+                    'log_name', self.log_route)})
         except (OSError, IOError) as err:
             if err.errno != errno.ENOENT and \
                     not str(err).endswith(' not found'):
                 raise
             raise SystemExit(
-                _('Unable to load internal client from config: '
-                  '%(conf)r (%(error)s)')
+                'Unable to load internal client from config: '
+                '%(conf)r (%(error)s)'
                 % {'conf': internal_client_conf_path, 'error': err})
 
     def run_forever(self, *args, **kwargs):
@@ -268,7 +247,7 @@ class ContainerSync(Daemon):
         """
         Runs a single container sync scan.
         """
-        self.logger.info(_('Begin container sync "once" mode'))
+        self.logger.info('Begin container sync "once" mode')
         begin = time()
         for path in self.sync_store.synced_containers_generator():
             self.container_sync(path)
@@ -277,7 +256,7 @@ class ContainerSync(Daemon):
         self.report()
         elapsed = time() - begin
         self.logger.info(
-            _('Container sync "once" mode completed: %.02fs'), elapsed)
+            'Container sync "once" mode completed: %.02fs', elapsed)
 
     def report(self):
         """
@@ -285,8 +264,8 @@ class ContainerSync(Daemon):
         next report.
         """
         self.logger.info(
-            _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s '
-              'puts], %(skip)s skipped, %(fail)s failed'),
+            'Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s '
+            'puts], %(skip)s skipped, %(fail)s failed',
             {'time': ctime(self.reported),
              'sync': self.container_syncs,
              'delete': self.container_deletes,
@@ -302,16 +281,16 @@ class ContainerSync(Daemon):
 
     def container_report(self, start, end, sync_point1, sync_point2, info,
                          max_row):
-        self.logger.info(_('Container sync report: %(container)s, '
-                           'time window start: %(start)s, '
-                           'time window end: %(end)s, '
-                           'puts: %(puts)s, '
-                           'posts: %(posts)s, '
-                           'deletes: %(deletes)s, '
-                           'bytes: %(bytes)s, '
-                           'sync_point1: %(point1)s, '
-                           'sync_point2: %(point2)s, '
-                           'total_rows: %(total)s'),
+        self.logger.info('Container sync report: %(container)s, '
+                         'time window start: %(start)s, '
+                         'time window end: %(end)s, '
+                         'puts: %(puts)s, '
+                         'posts: %(posts)s, '
+                         'deletes: %(deletes)s, '
+                         'bytes: %(bytes)s, '
+                         'sync_point1: %(point1)s, '
+                         'sync_point2: %(point2)s, '
+                         'total_rows: %(total)s',
                          {'container': '%s/%s' % (info['account'],
                                                   info['container']),
                           'start': start,
@@ -357,6 +336,13 @@ class ContainerSync(Daemon):
                     break
             else:
                 return
+            if broker.metadata.get(SYSMETA_VERSIONS_CONT):
+                self.container_skips += 1
+                self.logger.increment('skips')
+                self.logger.warning('Skipping container %s/%s with '
+                                    'object versioning configured' % (
+                                        info['account'], info['container']))
+                return
             if not broker.is_deleted():
                 sync_to = None
                 user_key = None
@@ -375,7 +361,7 @@ class ContainerSync(Daemon):
                     sync_to, self.allowed_sync_hosts, self.realms_conf)
                 if err:
                     self.logger.info(
-                        _('ERROR %(db_file)s: %(validate_sync_to_err)s'),
+                        'ERROR %(db_file)s: %(validate_sync_to_err)s',
                         {'db_file': str(broker),
                          'validate_sync_to_err': err})
                     self.container_failures += 1
@@ -444,7 +430,7 @@ class ContainerSync(Daemon):
         except (Exception, Timeout):
             self.container_failures += 1
             self.logger.increment('failures')
-            self.logger.exception(_('ERROR Syncing %s'),
+            self.logger.exception('ERROR Syncing %s',
                                   broker if broker else path)
 
     def _update_sync_to_headers(self, name, sync_to, user_key,
@@ -560,7 +546,8 @@ class ContainerSync(Daemon):
                                   logger=self.logger,
                                   timeout=self.conn_timeout)
                 except ClientException as err:
-                    if err.http_status != HTTP_NOT_FOUND:
+                    if err.http_status not in (
+                            HTTP_NOT_FOUND, HTTP_CONFLICT):
                         raise
                 self.container_deletes += 1
                 self.container_stats['deletes'] += 1
@@ -593,13 +580,23 @@ class ContainerSync(Daemon):
                     headers = {}
                     body = None
                     exc = err
+
+                # skip object_versioning links; this is in case the container
+                # metadata is out of date
+                if headers.get(SYSMETA_VERSIONS_SYMLINK):
+                    self.logger.info(
+                        'Skipping versioning symlink %s/%s/%s ' % (
+                            info['account'], info['container'],
+                            row['name']))
+                    return True
+
                 timestamp = Timestamp(headers.get('x-timestamp', 0))
                 if timestamp < ts_meta:
                     if exc:
                         raise exc
                     raise Exception(
-                        _('Unknown exception trying to GET: '
-                          '%(account)r %(container)r %(object)r'),
+                        'Unknown exception trying to GET: '
+                        '%(account)r %(container)r %(object)r' %
                         {'account': info['account'],
                          'container': info['container'],
                          'object': row['name']})
@@ -607,7 +604,7 @@ class ContainerSync(Daemon):
                     if key in headers:
                         del headers[key]
                 if 'etag' in headers:
-                    headers['etag'] = headers['etag'].strip('"')
+                    headers['etag'] = normalize_etag(headers['etag'])
                 if 'content-type' in headers:
                     headers['content-type'] = clean_content_type(
                         headers['content-type'])
@@ -625,27 +622,27 @@ class ContainerSync(Daemon):
         except ClientException as err:
             if err.http_status == HTTP_UNAUTHORIZED:
                 self.logger.info(
-                    _('Unauth %(sync_from)r => %(sync_to)r'),
+                    'Unauth %(sync_from)r => %(sync_to)r',
                     {'sync_from': '%s/%s' %
                         (quote(info['account']), quote(info['container'])),
                      'sync_to': sync_to})
             elif err.http_status == HTTP_NOT_FOUND:
                 self.logger.info(
-                    _('Not found %(sync_from)r => %(sync_to)r \
-                      - object %(obj_name)r'),
+                    'Not found %(sync_from)r => %(sync_to)r \
+                    - object %(obj_name)r',
                     {'sync_from': '%s/%s' %
                         (quote(info['account']), quote(info['container'])),
                      'sync_to': sync_to, 'obj_name': row['name']})
             else:
                 self.logger.exception(
-                    _('ERROR Syncing %(db_file)s %(row)s'),
+                    'ERROR Syncing %(db_file)s %(row)s',
                     {'db_file': str(broker), 'row': row})
             self.container_failures += 1
             self.logger.increment('failures')
             return False
-        except (Exception, Timeout) as err:
+        except (Exception, Timeout):
             self.logger.exception(
-                _('ERROR Syncing %(db_file)s %(row)s'),
+                'ERROR Syncing %(db_file)s %(row)s',
                 {'db_file': str(broker), 'row': row})
             self.container_failures += 1
             self.logger.increment('failures')
@@ -654,3 +651,12 @@ class ContainerSync(Daemon):
 
     def select_http_proxy(self):
         return choice(self.http_proxies) if self.http_proxies else None
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerSync, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/updater.py

@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import logging
 import os
 import signal
 import sys
 import time
-from swift import gettext_ as _
 from random import random, shuffle
 from tempfile import mkstemp
 
@@ -32,10 +30,11 @@ from swift.common.bufferedhttp import http_connect
 from swift.common.exceptions import ConnectionTimeout, LockTimeout
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, config_true_value, \
-    dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \
-    eventlet_monkey_patch
-from swift.common.daemon import Daemon
+    dump_recon_cache, majority_size, Timestamp, EventletRateLimiter, \
+    eventlet_monkey_patch, node_to_string, parse_options
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
+from swift.common.recon import RECON_CONTAINER_FILE, DEFAULT_RECON_CACHE_PATH
 
 
 class ContainerUpdater(Daemon):
@@ -47,7 +46,7 @@ class ContainerUpdater(Daemon):
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
         self.swift_dir = conf.get('swift_dir', '/etc/swift')
-        self.interval = int(conf.get('interval', 300))
+        self.interval = float(conf.get('interval', 300))
         self.account_ring = None
         self.concurrency = int(conf.get('concurrency', 4))
         if 'slowdown' in conf:
@@ -59,10 +58,10 @@ class ContainerUpdater(Daemon):
                 float(conf.get('slowdown', '0.01')) + 0.01)
         else:
             containers_per_second = 50
-        self.containers_running_time = 0
         self.max_containers_per_second = \
             float(conf.get('containers_per_second',
                            containers_per_second))
+        self.rate_limiter = EventletRateLimiter(self.max_containers_per_second)
         self.node_timeout = float(conf.get('node_timeout', 3))
         self.conn_timeout = float(conf.get('conn_timeout', 0.5))
         self.no_changes = 0
@@ -75,8 +74,8 @@ class ContainerUpdater(Daemon):
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
-        self.rcache = os.path.join(self.recon_cache_path, "container.recon")
+                                         DEFAULT_RECON_CACHE_PATH)
+        self.rcache = os.path.join(self.recon_cache_path, RECON_CONTAINER_FILE)
         self.user_agent = 'container-updater %s' % os.getpid()
 
     def get_account_ring(self):
@@ -89,8 +88,8 @@ class ContainerUpdater(Daemon):
         try:
             return os.listdir(path)
         except OSError as e:
-            self.logger.error(_('ERROR:  Failed to get paths to drive '
-                                'partitions: %s') % e)
+            self.logger.error('ERROR:  Failed to get paths to drive '
+                              'partitions: %s', e)
             return []
 
     def get_paths(self):
@@ -123,7 +122,7 @@ class ContainerUpdater(Daemon):
                     self.account_suppressions[account] = until
         except Exception:
             self.logger.exception(
-                _('ERROR with loading suppressions from %s: ') % filename)
+                'ERROR with loading suppressions from %s: ', filename)
         finally:
             os.unlink(filename)
 
@@ -133,7 +132,7 @@ class ContainerUpdater(Daemon):
         """
         time.sleep(random() * self.interval)
         while True:
-            self.logger.info(_('Begin container update sweep'))
+            self.logger.info('Begin container update sweep')
             begin = time.time()
             now = time.time()
             expired_suppressions = \
@@ -158,6 +157,7 @@ class ContainerUpdater(Daemon):
                     pid2filename[pid] = tmpfilename
                 else:
                     signal.signal(signal.SIGTERM, signal.SIG_DFL)
+                    os.environ.pop('NOTIFY_SOCKET', None)
                     eventlet_monkey_patch()
                     self.no_changes = 0
                     self.successes = 0
@@ -167,9 +167,9 @@ class ContainerUpdater(Daemon):
                     self.container_sweep(path)
                     elapsed = time.time() - forkbegin
                     self.logger.debug(
-                        _('Container update sweep of %(path)s completed: '
-                          '%(elapsed).02fs, %(success)s successes, %(fail)s '
-                          'failures, %(no_change)s with no changes'),
+                        'Container update sweep of %(path)s completed: '
+                        '%(elapsed).02fs, %(success)s successes, %(fail)s '
+                        'failures, %(no_change)s with no changes',
                         {'path': path, 'elapsed': elapsed,
                          'success': self.successes, 'fail': self.failures,
                          'no_change': self.no_changes})
@@ -181,7 +181,7 @@ class ContainerUpdater(Daemon):
                 finally:
                     del pid2filename[pid]
             elapsed = time.time() - begin
-            self.logger.info(_('Container update sweep completed: %.02fs'),
+            self.logger.info('Container update sweep completed: %.02fs',
                              elapsed)
             dump_recon_cache({'container_updater_sweep': elapsed},
                              self.rcache, self.logger)
@@ -193,7 +193,7 @@ class ContainerUpdater(Daemon):
         Run the updater once.
         """
         eventlet_monkey_patch()
-        self.logger.info(_('Begin container update single threaded sweep'))
+        self.logger.info('Begin container update single threaded sweep')
         begin = time.time()
         self.no_changes = 0
         self.successes = 0
@@ -201,10 +201,10 @@ class ContainerUpdater(Daemon):
         for path in self.get_paths():
             self.container_sweep(path)
         elapsed = time.time() - begin
-        self.logger.info(_(
+        self.logger.info(
             'Container update single threaded sweep completed: '
             '%(elapsed).02fs, %(success)s successes, %(fail)s failures, '
-            '%(no_change)s with no changes'),
+            '%(no_change)s with no changes',
             {'elapsed': elapsed, 'success': self.successes,
              'fail': self.failures, 'no_change': self.no_changes})
         dump_recon_cache({'container_updater_sweep': elapsed},
@@ -226,9 +226,7 @@ class ContainerUpdater(Daemon):
                         self.logger.exception(
                             "Error processing container %s: %s", dbfile, e)
 
-                    self.containers_running_time = ratelimit_sleep(
-                        self.containers_running_time,
-                        self.max_containers_per_second)
+                    self.rate_limiter.wait()
 
     def process_container(self, dbfile):
         """
@@ -270,23 +268,36 @@ class ContainerUpdater(Daemon):
                             info['storage_policy_index'])
                       for node in nodes]
             successes = 0
+            stub404s = 0
             for event in events:
-                if is_success(event.wait()):
+                result = event.wait()
+                if is_success(result):
                     successes += 1
+                if result == 404:
+                    stub404s += 1
             if successes >= majority_size(len(events)):
                 self.logger.increment('successes')
                 self.successes += 1
                 self.logger.debug(
-                    _('Update report sent for %(container)s %(dbfile)s'),
+                    'Update report sent for %(container)s %(dbfile)s',
                     {'container': container, 'dbfile': dbfile})
                 broker.reported(info['put_timestamp'],
                                 info['delete_timestamp'], info['object_count'],
                                 info['bytes_used'])
+            elif stub404s == len(events):
+                self.logger.increment('failures')
+                self.failures += 1
+                self.logger.debug(
+                    'Update report stub for %(container)s %(dbfile)s',
+                    {'container': container, 'dbfile': dbfile})
+                broker.quarantine('no account replicas exist')
+                # All that's left at this point is a few sacks of Gnocchi,
+                # easily collected by the dark data watcher in object auditor.
             else:
                 self.logger.increment('failures')
                 self.failures += 1
                 self.logger.debug(
-                    _('Update report failed for %(container)s %(dbfile)s'),
+                    'Update report failed for %(container)s %(dbfile)s',
                     {'container': container, 'dbfile': dbfile})
                 self.account_suppressions[info['account']] = until = \
                     time.time() + self.account_suppression_time
@@ -325,12 +336,12 @@ class ContainerUpdater(Daemon):
                     'X-Backend-Storage-Policy-Index': storage_policy_index,
                     'user-agent': self.user_agent}
                 conn = http_connect(
-                    node['ip'], node['port'], node['device'], part,
-                    'PUT', container, headers=headers)
+                    node['replication_ip'], node['replication_port'],
+                    node['device'], part, 'PUT', container, headers=headers)
             except (Exception, Timeout):
-                self.logger.exception(_(
-                    'ERROR account update failed with '
-                    '%(ip)s:%(port)s/%(device)s (will retry later): '), node)
+                self.logger.exception(
+                    'ERROR account update failed with %s (will retry later):',
+                    node_to_string(node, replication=True))
                 return HTTP_INTERNAL_SERVER_ERROR
         with Timeout(self.node_timeout):
             try:
@@ -340,7 +351,17 @@ class ContainerUpdater(Daemon):
             except (Exception, Timeout):
                 if self.logger.getEffectiveLevel() <= logging.DEBUG:
                     self.logger.exception(
-                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
+                        'Exception with %s',
+                        node_to_string(node, replication=True))
                 return HTTP_INTERNAL_SERVER_ERROR
             finally:
                 conn.close()
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerUpdater, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()