swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/db_auditor.py

@@ -0,0 +1,168 @@
+# Copyright (c) 2010-2018 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from random import random
+
+from eventlet import Timeout
+
+import swift.common.db
+from swift.common.utils import get_logger, audit_location_generator, \
+    config_true_value, dump_recon_cache, EventletRateLimiter
+from swift.common.daemon import Daemon
+from swift.common.exceptions import DatabaseAuditorException
+from swift.common.recon import DEFAULT_RECON_CACHE_PATH, \
+    server_type_to_recon_file
+
+
+class DatabaseAuditor(Daemon):
+    """Base Database Auditor."""
+
+    @property
+    def rcache(self):
+        return os.path.join(
+            self.recon_cache_path,
+            server_type_to_recon_file(self.server_type))
+
+    @property
+    def server_type(self):
+        raise NotImplementedError
+
+    @property
+    def broker_class(self):
+        raise NotImplementedError
+
+    def __init__(self, conf, logger=None):
+        self.conf = conf
+        self.logger = logger or get_logger(conf, log_route='{}-auditor'.format(
+            self.server_type))
+        self.devices = conf.get('devices', '/srv/node')
+        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
+        self.interval = float(conf.get('interval', 1800))
+        self.logging_interval = 3600  # once an hour
+        self.passes = 0
+        self.failures = 0
+        self.max_dbs_per_second = \
+            float(conf.get('{}s_per_second'.format(self.server_type), 200))
+        self.rate_limiter = EventletRateLimiter(self.max_dbs_per_second)
+        swift.common.db.DB_PREALLOCATION = \
+            config_true_value(conf.get('db_preallocation', 'f'))
+        self.recon_cache_path = conf.get('recon_cache_path',
+                                         DEFAULT_RECON_CACHE_PATH)
+        self.datadir = '{}s'.format(self.server_type)
+
+    def _one_audit_pass(self, reported):
+        all_locs = audit_location_generator(self.devices, self.datadir, '.db',
+                                            mount_check=self.mount_check,
+                                            logger=self.logger)
+        for path, device, partition in all_locs:
+            self.audit(path)
+            if time.time() - reported >= self.logging_interval:
+                self.logger.info(
+                    'Since %(time)s: %(server_type)s audits: %(pass)s '
+                    'passed audit, %(fail)s failed audit',
+                    {'time': time.ctime(reported),
+                     'pass': self.passes,
+                     'fail': self.failures,
+                     'server_type': self.server_type})
+                dump_recon_cache(
+                    {'{}_audits_since'.format(self.server_type): reported,
+                     '{}_audits_passed'.format(self.server_type): self.passes,
+                     '{}_audits_failed'.format(self.server_type):
+                         self.failures},
+                    self.rcache, self.logger)
+                reported = time.time()
+                self.passes = 0
+                self.failures = 0
+            self.rate_limiter.wait()
+        return reported
+
+    def run_forever(self, *args, **kwargs):
+        """Run the database audit until stopped."""
+        reported = time.time()
+        time.sleep(random() * self.interval)
+        while True:
+            self.logger.info(
+                'Begin %s audit pass.', self.server_type)
+            begin = time.time()
+            try:
+                reported = self._one_audit_pass(reported)
+            except (Exception, Timeout):
+                self.logger.increment('errors')
+                self.logger.exception('ERROR auditing')
+            elapsed = time.time() - begin
+            self.logger.info(
+                '%(server_type)s audit pass completed: %(elapsed).02fs',
+                {'elapsed': elapsed, 'server_type': self.server_type.title()})
+            dump_recon_cache({
+                '{}_auditor_pass_completed'.format(self.server_type): elapsed},
+                self.rcache, self.logger)
+            if elapsed < self.interval:
+                time.sleep(self.interval - elapsed)
+
+    def run_once(self, *args, **kwargs):
+        """Run the database audit once."""
+        self.logger.info(
+            'Begin %s audit "once" mode', self.server_type)
+        begin = reported = time.time()
+        self._one_audit_pass(reported)
+        elapsed = time.time() - begin
+        self.logger.info(
+            '%(server_type)s audit "once" mode completed: %(elapsed).02fs',
+            {'elapsed': elapsed, 'server_type': self.server_type.title()})
+        dump_recon_cache(
+            {'{}_auditor_pass_completed'.format(self.server_type): elapsed},
+            self.rcache, self.logger)
+
+    def audit(self, path):
+        """
+        Audits the given database path
+
+        :param path: the path to a db
+        """
+        start_time = time.time()
+        try:
+            broker = self.broker_class(path, logger=self.logger)
+            if not broker.is_deleted():
+                info = broker.get_info()
+                err = self._audit(info, broker)
+                if err:
+                    raise err
+                self.logger.increment('passes')
+                self.passes += 1
+                self.logger.debug('Audit passed for %s', broker)
+        except DatabaseAuditorException as e:
+            self.logger.increment('failures')
+            self.failures += 1
+            self.logger.error('Audit Failed for %(path)s: %(err)s',
+                              {'path': path, 'err': str(e)})
+        except (Exception, Timeout):
+            self.logger.increment('failures')
+            self.failures += 1
+            self.logger.exception(
+                'ERROR Could not get %(server_type)s info %(path)s',
+                {'server_type': self.server_type, 'path': path})
+        self.logger.timing_since('timing', start_time)
+
+    def _audit(self, info, broker):
+        """
+        Run any additional audit checks in sub auditor classes
+
+        :param info: The DB <account/container>_info
+        :param broker: The broker
+        :return: None on success, otherwise an exception to throw.
+        """
+        raise NotImplementedError

swift/common/db_replicator.py

@@ -23,7 +23,6 @@ import uuid
 import errno
 import re
 from contextlib import contextmanager
-from swift import gettext_ as _
 
 from eventlet import GreenPool, sleep, Timeout
 from eventlet.green import subprocess
@@ -34,7 +33,7 @@ from swift.common.utils import get_logger, whataremyips, storage_directory, \
     renamer, mkdirs, lock_parent_directory, config_true_value, \
     unlink_older_than, dump_recon_cache, rsync_module_interpolation, \
     parse_override_options, round_robin_iter, Everything, get_db_files, \
-    parse_db_filename, quote, RateLimitedIterator
+    parse_db_filename, quote, RateLimitedIterator, config_auto_int_value
 from swift.common import ring
 from swift.common.ring.utils import is_local_device
 from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE, \
@@ -44,6 +43,8 @@ from swift.common.exceptions import DriveNotMounted
 from swift.common.daemon import Daemon
 from swift.common.swob import Response, HTTPNotFound, HTTPNoContent, \
     HTTPAccepted, HTTPBadRequest
+from swift.common.recon import DEFAULT_RECON_CACHE_PATH, \
+    server_type_to_recon_file
 
 
 DEBUG_TIMINGS_THRESHOLD = 10
@@ -172,8 +173,9 @@ class ReplConnection(BufferedHTTPConnection):
             response.data = response.read()
             return response
         except (Exception, Timeout):
+            self.close()
             self.logger.exception(
-                _('ERROR reading HTTP response from %s'), self.node)
+                'ERROR reading HTTP response from %s', self.node)
             return None
 
 
@@ -193,19 +195,27 @@ class Replicator(Daemon):
         self.cpool = GreenPool(size=concurrency)
         swift_dir = conf.get('swift_dir', '/etc/swift')
         self.ring = ring.Ring(swift_dir, ring_name=self.server_type)
-        self._local_device_ids = set()
+        self._local_device_ids = {}
         self.per_diff = int(conf.get('per_diff', 1000))
         self.max_diffs = int(conf.get('max_diffs') or 100)
-        self.interval = int(conf.get('interval') or
-                            conf.get('run_pause') or 30)
-        if 'run_pause' in conf and 'interval' not in conf:
-            self.logger.warning('Option %(type)s-replicator/run_pause '
-                                'is deprecated and will be removed in a '
-                                'future version. Update your configuration'
-                                ' to use option %(type)s-replicator/'
-                                'interval.'
-                                % {'type': self.server_type})
-        self.databases_per_second = int(
+        self.interval = float(conf.get('interval') or
+                              conf.get('run_pause') or 30)
+        if 'run_pause' in conf:
+            if 'interval' in conf:
+                self.logger.warning(
+                    'Option %(type)s-replicator/run_pause is deprecated '
+                    'and %(type)s-replicator/interval is already configured. '
+                    'You can safely remove run_pause; it is now ignored and '
+                    'will be removed in a future version.'
+                    % {'type': self.server_type})
+            else:
+                self.logger.warning(
+                    'Option %(type)s-replicator/run_pause is deprecated '
+                    'and will be removed in a future version. '
+                    'Update your configuration to use option '
+                    '%(type)s-replicator/interval.'
+                    % {'type': self.server_type})
+        self.databases_per_second = float(
             conf.get('databases_per_second', 50))
         self.node_timeout = float(conf.get('node_timeout', 10))
         self.conn_timeout = float(conf.get('conn_timeout', 0.5))
@@ -217,15 +227,25 @@ class Replicator(Daemon):
         self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
+        swift.common.db.QUERY_LOGGING = \
+            config_true_value(conf.get('db_query_logging', 'f'))
         self._zero_stats()
         self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
-        self.recon_replicator = '%s.recon' % self.server_type
+                                         DEFAULT_RECON_CACHE_PATH)
+        self.recon_replicator = server_type_to_recon_file(self.server_type)
         self.rcache = os.path.join(self.recon_cache_path,
                                    self.recon_replicator)
         self.extract_device_re = re.compile('%s%s([^%s]+)' % (
             self.root, os.path.sep, os.path.sep))
         self.handoffs_only = config_true_value(conf.get('handoffs_only', 'no'))
+        self.handoff_delete = config_auto_int_value(
+            conf.get('handoff_delete', 'auto'), 0)
+        if self.handoff_delete >= self.ring.replica_count:
+            self.logger.warning(
+                'handoff_delete=%d is too high to have an effect on a ring '
+                'with replica count %d. Disabling.',
+                self.handoff_delete, self.ring.replica_count)
+            self.handoff_delete = 0
 
     def _zero_stats(self):
         """Zero out the stats."""
@@ -239,15 +259,15 @@ class Replicator(Daemon):
         """Report the current stats to the logs."""
         now = time.time()
         self.logger.info(
-            _('Attempted to replicate %(count)d dbs in %(time).5f seconds '
-              '(%(rate).5f/s)'),
+            'Attempted to replicate %(count)d dbs in %(time).5f seconds '
+            '(%(rate).5f/s)',
             {'count': self.stats['attempted'],
              'time': now - self.stats['start'],
              'rate': self.stats['attempted'] /
                 (now - self.stats['start'] + 0.0000001)})
-        self.logger.info(_('Removed %(remove)d dbs') % self.stats)
-        self.logger.info(_('%(success)s successes, %(failure)s failures')
-                         % self.stats)
+        self.logger.info('Removed %(remove)d dbs', self.stats)
+        self.logger.info('%(success)s successes, %(failure)s failures',
+                         self.stats)
         dump_recon_cache(
             {'replication_stats': self.stats,
              'replication_time': now - self.stats['start'],
@@ -293,7 +313,7 @@ class Replicator(Daemon):
         proc = subprocess.Popen(popen_args)
         proc.communicate()
         if proc.returncode != 0:
-            self.logger.error(_('ERROR rsync failed with %(code)s: %(args)s'),
+            self.logger.error('ERROR rsync failed with %(code)s: %(args)s',
                               {'code': proc.returncode, 'args': popen_args})
         return proc.returncode == 0
 
@@ -426,7 +446,7 @@ class Replicator(Daemon):
         Make an http_connection using ReplConnection
 
         :param node: node dictionary from the ring
-        :param partition: partition partition to send in the url
+        :param partition: partition to send in the url
         :param db_file: DB file
 
         :returns: ReplConnection object
@@ -542,7 +562,13 @@ class Replicator(Daemon):
             reason = '%s new rows' % max_row_delta
             self.logger.debug(log_template, reason)
             return True
-        if not (responses and all(responses)):
+        if self.handoff_delete:
+            # delete handoff if we have had handoff_delete successes
+            successes_count = len([resp for resp in responses if resp])
+            delete_handoff = successes_count >= self.handoff_delete
+        else:
+            delete_handoff = responses and all(responses)
+        if not delete_handoff:
             reason = '%s/%s success' % (responses.count(True), len(responses))
             self.logger.debug(log_template, reason)
             return True
@@ -556,6 +582,12 @@ class Replicator(Daemon):
         self.logger.debug('Successfully deleted db %s', broker.db_file)
         return True
 
+    def _reclaim(self, broker, now=None):
+        if not now:
+            now = time.time()
+        return broker.reclaim(now - self.reclaim_age,
+                              now - (self.reclaim_age * 2))
+
     def _replicate_object(self, partition, object_file, node_id):
         """
         Replicate the db, choosing method based on whether or not it
@@ -579,9 +611,9 @@ class Replicator(Daemon):
         shouldbehere = True
         responses = []
         try:
-            broker = self.brokerclass(object_file, pending_timeout=30)
-            broker.reclaim(now - self.reclaim_age,
-                           now - (self.reclaim_age * 2))
+            broker = self.brokerclass(object_file, pending_timeout=30,
+                                      logger=self.logger)
+            self._reclaim(broker, now)
             info = broker.get_replication_info()
             bpart = self.ring.get_part(
                 info['account'], info.get('container'))
@@ -598,10 +630,10 @@ class Replicator(Daemon):
                     'replicate out and remove.' % (object_file, name, bpart))
         except (Exception, Timeout) as e:
             if 'no such table' in str(e):
-                self.logger.error(_('Quarantining DB %s'), object_file)
+                self.logger.error('Quarantining DB %s', object_file)
                 quarantine_db(broker.db_file, broker.db_type)
             else:
-                self.logger.exception(_('ERROR reading db %s'), object_file)
+                self.logger.exception('ERROR reading db %s', object_file)
             nodes = self.ring.get_part_nodes(int(partition))
             self._add_failure_stats([(failure_dev['replication_ip'],
                                       failure_dev['device'])
@@ -653,13 +685,13 @@ class Replicator(Daemon):
                     repl_nodes.append(next(more_nodes))
                 except StopIteration:
                     self.logger.error(
-                        _('ERROR There are not enough handoff nodes to reach '
-                          'replica count for partition %s'),
+                        'ERROR There are not enough handoff nodes to reach '
+                        'replica count for partition %s',
                         partition)
-                self.logger.error(_('ERROR Remote drive not mounted %s'), node)
+                self.logger.error('ERROR Remote drive not mounted %s', node)
             except (Exception, Timeout):
-                self.logger.exception(_('ERROR syncing %(file)s with node'
-                                        ' %(node)s'),
+                self.logger.exception('ERROR syncing %(file)s with node'
+                                      ' %(node)s',
                                       {'file': object_file, 'node': node})
             if not success:
                 failure_devs_info.add((node['replication_ip'], node['device']))
@@ -692,16 +724,22 @@ class Replicator(Daemon):
         suf_dir = os.path.dirname(hash_dir)
         with lock_parent_directory(object_file):
             shutil.rmtree(hash_dir, True)
-        try:
-            os.rmdir(suf_dir)
-        except OSError as err:
-            if err.errno not in (errno.ENOENT, errno.ENOTEMPTY):
-                self.logger.exception(
-                    _('ERROR while trying to clean up %s') % suf_dir)
-                return False
         self.stats['remove'] += 1
         device_name = self.extract_device(object_file)
         self.logger.increment('removes.' + device_name)
+
+        for parent_dir in (suf_dir, os.path.dirname(suf_dir)):
+            try:
+                os.rmdir(parent_dir)
+            except OSError as err:
+                if err.errno == errno.ENOTEMPTY:
+                    break
+                elif err.errno == errno.ENOENT:
+                    continue
+                else:
+                    self.logger.exception(
+                        'ERROR while trying to clean up %s', parent_dir)
+                    return False
         return True
 
     def extract_device(self, object_file):
@@ -752,16 +790,17 @@ class Replicator(Daemon):
         dirs = []
         ips = whataremyips(self.bind_ip)
         if not ips:
-            self.logger.error(_('ERROR Failed to get my own IPs?'))
+            self.logger.error('ERROR Failed to get my own IPs?')
             return
 
-        if self.handoffs_only:
+        if self.handoffs_only or self.handoff_delete:
             self.logger.warning(
-                'Starting replication pass with handoffs_only enabled. '
-                'This mode is not intended for normal '
-                'operation; use handoffs_only with care.')
+                'Starting replication pass with handoffs_only '
+                'and/or handoffs_delete enabled. '
+                'These modes are not intended for normal '
+                'operation; use these options with care.')
 
-        self._local_device_ids = set()
+        self._local_device_ids = {}
         found_local = False
         for node in self.ring.devs:
             if node and is_local_device(ips, self.port,
@@ -788,7 +827,7 @@ class Replicator(Daemon):
                     time.time() - self.reclaim_age)
                 datadir = os.path.join(self.root, node['device'], self.datadir)
                 if os.path.isdir(datadir):
-                    self._local_device_ids.add(node['id'])
+                    self._local_device_ids[node['id']] = node
                     part_filt = self._partition_dir_filter(
                         node['id'], partitions_to_replicate)
                     dirs.append((datadir, node['id'], part_filt))
@@ -796,16 +835,17 @@ class Replicator(Daemon):
             self.logger.error("Can't find itself %s with port %s in ring "
                               "file, not replicating",
                               ", ".join(ips), self.port)
-        self.logger.info(_('Beginning replication run'))
+        self.logger.info('Beginning replication run')
         for part, object_file, node_id in self.roundrobin_datadirs(dirs):
             self.cpool.spawn_n(
                 self._replicate_object, part, object_file, node_id)
         self.cpool.waitall()
-        self.logger.info(_('Replication run OVER'))
-        if self.handoffs_only:
+        self.logger.info('Replication run OVER')
+        if self.handoffs_only or self.handoff_delete:
             self.logger.warning(
-                'Finished replication pass with handoffs_only enabled. '
-                'If handoffs_only is no longer required, disable it.')
+                'Finished replication pass with handoffs_only and/or '
+                'handoffs_delete enabled. If these are no longer required, '
+                'disable them.')
         self._report_stats()
 
     def run_forever(self, *args, **kwargs):
@@ -818,7 +858,7 @@ class Replicator(Daemon):
             try:
                 self.run_once()
             except (Exception, Timeout):
-                self.logger.exception(_('ERROR trying to replicate'))
+                self.logger.exception('ERROR trying to replicate')
             elapsed = time.time() - begin
             if elapsed < self.interval:
                 sleep(self.interval - elapsed)
@@ -922,7 +962,7 @@ class ReplicatorRpc(object):
                 info = self._get_synced_replication_info(broker, remote_info)
             except (Exception, Timeout) as e:
                 if 'no such table' in str(e):
-                    self.logger.error(_("Quarantining DB %s"), broker)
+                    self.logger.error("Quarantining DB %s", broker)
                     quarantine_db(broker.db_file, broker.db_type)
                     return HTTPNotFound()
                 raise

swift/common/digest.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2022 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import binascii
+import hmac
+
+from swift.common.utils import strict_b64decode
+
+
+DEFAULT_ALLOWED_DIGESTS = 'sha1 sha256 sha512'
+DEPRECATED_DIGESTS = {'sha1'}
+SUPPORTED_DIGESTS = set(DEFAULT_ALLOWED_DIGESTS.split()) | DEPRECATED_DIGESTS
+
+
+def get_hmac(request_method, path, expires, key, digest="sha1",
+             ip_range=None):
+    """
+    Returns the hexdigest string of the HMAC (see RFC 2104) for
+    the request.
+
+    :param request_method: Request method to allow.
+    :param path: The path to the resource to allow access to.
+    :param expires: Unix timestamp as an int for when the URL
+                    expires.
+    :param key: HMAC shared secret.
+    :param digest: constructor or the string name for the digest to use in
+                   calculating the HMAC
+                   Defaults to SHA1
+    :param ip_range: The ip range from which the resource is allowed
+                     to be accessed. We need to put the ip_range as the
+                     first argument to hmac to avoid manipulation of the path
+                     due to newlines being valid in paths
+                     e.g. /v1/a/c/o\\n127.0.0.1
+    :returns: hexdigest str of the HMAC for the request using the specified
+              digest algorithm.
+    """
+    # These are the three mandatory fields.
+    parts = [request_method, str(expires), path]
+    formats = [b"%s", b"%s", b"%s"]
+
+    if ip_range:
+        parts.insert(0, ip_range)
+        formats.insert(0, b"ip=%s")
+
+    if isinstance(key, str):
+        key = key.encode('utf8')
+
+    message = b'\n'.join(
+        fmt % (part if isinstance(part, bytes)
+               else part.encode("utf-8"))
+        for fmt, part in zip(formats, parts))
+
+    return hmac.new(key, message, digest).hexdigest()
+
+
+def get_allowed_digests(conf_digests, logger=None):
+    """
+    Pulls out 'allowed_digests' from the supplied conf. Then compares them with
+    the list of supported and deprecated digests and returns whatever remain.
+
+    When something is unsupported or deprecated it'll log a warning.
+
+    :param conf_digests: iterable of allowed digests. If empty, defaults to
+        DEFAULT_ALLOWED_DIGESTS.
+    :param logger: optional logger; if provided, use it issue deprecation
+        warnings
+    :returns: A set of allowed digests that are supported and a set of
+        deprecated digests.
+    :raises: ValueError, if there are no digests left to return.
+    """
+    allowed_digests = set(digest.lower() for digest in conf_digests)
+    if not allowed_digests:
+        allowed_digests = SUPPORTED_DIGESTS
+
+    not_supported = allowed_digests - SUPPORTED_DIGESTS
+    if not_supported:
+        if logger:
+            logger.warning('The following digest algorithms are configured '
+                           'but not supported: %s', ', '.join(not_supported))
+        allowed_digests -= not_supported
+    deprecated = allowed_digests & DEPRECATED_DIGESTS
+    if deprecated and logger:
+        if not conf_digests:
+            logger.warning('The following digest algorithms are allowed by '
+                           'default but deprecated: %s. Support will be '
+                           'disabled by default in a future release, and '
+                           'later removed entirely.', ', '.join(deprecated))
+        else:
+            logger.warning('The following digest algorithms are configured '
+                           'but deprecated: %s. Support will be removed in a '
+                           'future release.', ', '.join(deprecated))
+    if not allowed_digests:
+        raise ValueError('No valid digest algorithms are configured')
+
+    return allowed_digests, deprecated
+
+
+def extract_digest_and_algorithm(value):
+    """
+    Returns a tuple of (digest_algorithm, hex_encoded_digest)
+    from a client-provided string of the form::
+
+       <hex-encoded digest>
+
+    or::
+
+       <algorithm>:<base64-encoded digest>
+
+    Note that hex-encoded strings must use one of sha1, sha256, or sha512.
+
+    :raises: ValueError on parse failures
+    """
+    if ':' in value:
+        algo, value = value.split(':', 1)
+        # accept both standard and url-safe base64
+        if ('-' in value or '_' in value) and not (
+                '+' in value or '/' in value):
+            value = value.replace('-', '+').replace('_', '/')
+        value = binascii.hexlify(
+            strict_b64decode(value + '==')).decode('ascii')
+    else:
+        binascii.unhexlify(value)  # make sure it decodes
+        algo = {
+            40: 'sha1',
+            64: 'sha256',
+            128: 'sha512',
+        }.get(len(value))
+        if not algo:
+            raise ValueError('Bad digest length')
+    return algo, value