rucio 32.8.6__py3-none-any.whl

rucio/daemons/atropos/atropos.py

@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import functools
+import logging
+import random
+import threading
+from sys import exc_info
+from traceback import format_exception
+from typing import TYPE_CHECKING
+
+import rucio.core.lifetime_exception
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.exception import InvalidRSEExpression, RuleNotFound
+from rucio.common.logging import setup_logging
+from rucio.core.did import set_metadata
+from rucio.core.lock import get_dataset_locks
+from rucio.core.rse import get_rse_name, get_rse_vo
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.core.rule import get_rules_beyond_eol, update_rule
+from rucio.daemons.common import run_daemon
+from rucio.db.sqla.constants import LifetimeExceptionsState
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.daemons.common import HeartbeatHandler
+
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'atropos'
+
+
+def atropos(
+        date_check: datetime.datetime,
+        dry_run: bool,
+        grace_period: int,
+        purge_replicas: bool,
+        spread_period: bool,
+        unlock: bool,
+        once: bool,
+        sleep_time: int
+) -> None:
+    """
+    Creates an Atropos Worker that gets a list of rules which have an eol_at expired and delete them.
+
+    :param grace_period: The grace_period for the rules.
+    :param once: Run only once.
+    :param sleep_time: Thread sleep time after each chunk of work.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            date_check=date_check,
+            dry_run=dry_run,
+            grace_period=grace_period,
+            purge_replicas=purge_replicas,
+            spread_period=spread_period,
+            unlock=unlock
+        )
+    )
+
+
+def run_once(
+        heartbeat_handler: 'HeartbeatHandler',
+        activity: None,    # NOQA, pylint: disable=W0613
+        date_check: datetime.datetime,
+        dry_run: bool,
+        grace_period: int,
+        purge_replicas: bool,
+        spread_period: int,
+        unlock: bool,
+) -> None:
+    """An iteration of an Atropos worker."""
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.DEBUG, 'Starting worker')
+    if not dry_run and date_check > datetime.datetime.utcnow():
+        logger(logging.ERROR, 'Atropos cannot run in non-dry-run mode for date in the future')
+        return
+
+    # Process the list of approved exceptions. In case a DID has
+    # multiple exceptions, the one with the expiration date further in
+    # the future is what matters.
+    summary = {}
+    lifetime_exceptions = {}
+    for excep in rucio.core.lifetime_exception.list_exceptions(exception_id=None, states=[LifetimeExceptionsState.APPROVED, ], session=None):
+        key = '{}:{}'.format(excep['scope'].internal, excep['name'])
+        if key not in lifetime_exceptions:
+            lifetime_exceptions[key] = excep['expires_at']
+        elif lifetime_exceptions[key] < excep['expires_at']:
+            lifetime_exceptions[key] = excep['expires_at']
+    logger(logging.DEBUG, '%d active exceptions', len(lifetime_exceptions))
+
+    rand = random.Random(worker_number)
+
+    try:
+        rules = get_rules_beyond_eol(date_check, worker_number, total_workers, session=None)
+        logger(logging.INFO, '%d rules to process', len(rules))
+        for rule_idx, rule in enumerate(rules, start=1):
+            did = '%s:%s' % (rule.scope, rule.name)
+            did_key = '{}:{}'.format(rule.scope.internal, rule.name)
+            logger(logging.DEBUG, 'Working on rule %s on DID %s on %s', rule.id, did, rule.rse_expression)
+
+            if (rule_idx % 1000) == 0:
+                logger(logging.INFO, '%s/%s rules processed', rule_idx, len(rules))
+
+            # We compute the expected eol_at
+            try:
+                rses = parse_expression(rule.rse_expression, filter_={'vo': rule.account.vo})
+            except InvalidRSEExpression:
+                logger(logging.WARNING, 'Rule %s has an RSE expression that results in an empty set: %s', rule.id, rule.rse_expression)
+                continue
+            eol_at = rucio.core.lifetime_exception.define_eol(rule.scope, rule.name, rses)
+            if eol_at != rule.eol_at:
+                logger(logging.WARNING, 'The computed eol %s differs from the one recorded %s for rule %s on %s at %s',
+                       eol_at, rule.eol_at, rule.id, did, rule.rse_expression)
+                try:
+                    update_rule(rule.id, options={'eol_at': eol_at})
+                except RuleNotFound:
+                    logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                    continue
+
+            # Check the exceptions
+            if did_key in lifetime_exceptions:
+                if eol_at > lifetime_exceptions[did_key]:
+                    logger(logging.INFO, 'Rule %s on DID %s on %s has longer expiration date than the one requested : %s',
+                           rule.id, did, rule.rse_expression, lifetime_exceptions[did_key])
+                else:
+                    # If eol_at < requested extension, update eol_at
+                    logger(logging.INFO, 'Updating rule %s on DID %s on %s according to the exception till %s',
+                           rule.id, did, rule.rse_expression, lifetime_exceptions[did_key])
+                    eol_at = lifetime_exceptions[did_key]
+                    try:
+                        update_rule(rule.id, options={'eol_at': lifetime_exceptions[did_key]})
+                    except RuleNotFound:
+                        logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                        continue
+
+            # Now check that the new eol_at is expired
+            if eol_at and eol_at <= date_check:
+                set_metadata(scope=rule.scope, name=rule.name, key='eol_at', value=eol_at)
+                no_locks = True
+                for lock in get_dataset_locks(rule.scope, rule.name):
+                    if lock['rule_id'] == rule[4]:
+                        no_locks = False
+                        if lock['rse_id'] not in summary:
+                            summary[lock['rse_id']] = {}
+                        if did_key not in summary[lock['rse_id']]:
+                            summary[lock['rse_id']][did_key] = {'length': lock['length'] or 0, 'bytes': lock['bytes'] or 0}
+                if no_locks:
+                    logger(logging.WARNING, 'Cannot find a lock for rule %s on DID %s', rule.id, did)
+                if not dry_run:
+                    lifetime = grace_period + rand.randrange(spread_period + 1)
+                    logger(logging.INFO, 'Setting %s seconds lifetime for rule %s', lifetime, rule.id)
+                    options = {'lifetime': lifetime}
+                    if purge_replicas:
+                        options['purge_replicas'] = True
+                    if rule.locked and unlock:
+                        logger(logging.INFO, 'Unlocking rule %s', rule.id)
+                        options['locked'] = False
+                    try:
+                        update_rule(rule.id, options=options)
+                    except RuleNotFound:
+                        logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                        continue
+    except Exception:
+        exc_type, exc_value, exc_traceback = exc_info()
+        logger(logging.CRITICAL, ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())
+
+    for rse_id in summary:
+        tot_size, tot_files, tot_datasets = 0, 0, 0
+        for did in summary[rse_id]:
+            tot_datasets += 1
+            tot_files += summary[rse_id][did].get('length', 0)
+            tot_size += summary[rse_id][did].get('bytes', 0)
+        vo = get_rse_vo(rse_id=rse_id)
+        logger(logging.INFO, 'For RSE %s%s %d datasets will be deleted representing %d files and %d bytes',
+               get_rse_name(rse_id=rse_id), '' if vo == 'def' else ' on VO ' + vo, tot_datasets, tot_files, tot_size)
+
+
+def run(
+        date_check: datetime.datetime = datetime.datetime.utcnow(),
+        dry_run: bool = True,
+        grace_period: int = 86400,
+        purge_replicas: bool = False,
+        spread_period: int = 0,
+        unlock: bool = False,
+        once: bool = True,
+        sleep_time: int = 60,
+        threads: int = 1,
+) -> None:
+    """
+    Starts up the atropos threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('Will run only one iteration')
+    logging.info('starting atropos threads')
+    thread_list = [threading.Thread(target=atropos, kwargs={'date_check': date_check,
+                                                            'dry_run': dry_run,
+                                                            'grace_period': grace_period,
+                                                            'purge_replicas': purge_replicas,
+                                                            'spread_period': spread_period,
+                                                            'unlock': unlock,
+                                                            'once': once,
+                                                            'sleep_time': sleep_time}) for i in range(0, threads)]
+    [t.start() for t in thread_list]
+
+    logging.info('waiting for interrupts')
+
+    # Interruptible joins require a timeout.
+    while thread_list:
+        thread_list = [t.join(timeout=3.14) for t in thread_list if t and t.is_alive()]
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    GRACEFUL_STOP.set()

rucio/daemons/auditor/__init__.py

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import bz2
+import glob
+import logging
+import os
+import queue as Queue
+from datetime import datetime
+from datetime import timedelta
+
+import select
+
+from rucio.common import config
+from rucio.common.dumper import LogPipeHandler
+from rucio.common.dumper import mkdir
+from rucio.common.dumper import temp_file
+from rucio.common.dumper.consistency import Consistency
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.utils import chunks
+from rucio.core.quarantined_replica import add_quarantined_replicas
+from rucio.core.replica import declare_bad_file_replicas, list_replicas
+from rucio.core.rse import get_rse_usage, get_rse_id
+from rucio.daemons.auditor import srmdumps
+from rucio.daemons.auditor.hdfs import ReplicaFromHDFS
+from rucio.db.sqla.constants import BadFilesStatus
+
+
+def consistency(rse, delta, configuration, cache_dir, results_dir):
+    logger = logging.getLogger('auditor-worker')
+    rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
+    results_path = os.path.join(results_dir, '{0}_{1}'.format(rse, rsedate.strftime('%Y%m%d')))  # pylint: disable=no-member
+
+    if os.path.exists(results_path + '.bz2') or os.path.exists(results_path):
+        logger.warning('Consistency check for "%s" (dump dated %s) already done, skipping check', rse, rsedate.strftime('%Y%m%d'))  # pylint: disable=no-member
+        return None
+
+    rrdump_prev = ReplicaFromHDFS.download(rse, rsedate - delta, cache_dir=cache_dir)
+    rrdump_next = ReplicaFromHDFS.download(rse, rsedate + delta, cache_dir=cache_dir)
+    results = Consistency.dump(
+        'consistency-manual',
+        rse,
+        rsedump,
+        rrdump_prev,
+        rrdump_next,
+        date=rsedate,
+        cache_dir=cache_dir,
+    )
+    mkdir(results_dir)
+    with temp_file(results_dir, results_path) as (output, _):
+        for result in results:
+            output.write('{0}\n'.format(result.csv()))
+
+    return results_path
+
+
+def guess_replica_info(path):
+    """Try to extract the scope and name from a path.
+
+    ``path`` should be an ``str`` with the relative path to the file on
+    the RSE.
+
+    Returns a ``tuple`` of which the first element is the scope of the
+    replica and the second element is its name.
+    """
+    items = path.split('/')
+    if len(items) == 1:
+        return None, path
+    elif len(items) > 2 and items[0] in ['group', 'user']:
+        return '.'.join(items[0:2]), items[-1]
+    else:
+        return items[0], items[-1]
+
+
+def bz2_compress_file(source, chunk_size=65000):
+    """Compress a file with bzip2.
+
+    The destination is the path passed through ``source`` extended with
+    '.bz2'.  The original file is deleted.
+
+    Errors are deliberately not handled gracefully.  Any exceptions
+    should be propagated to the caller.
+
+    ``source`` should be an ``str`` with the absolute path to the file
+    to compress.
+
+    ``chunk_size`` should be an ``int`` with the size (in bytes) of the
+    chunks by which to read the file.
+
+    Returns an ``str`` with the destination path.
+    """
+    destination = '{}.bz2'.format(source)
+    with open(source) as plain, bz2.BZ2File(destination, 'w') as compressed:
+        while True:
+            chunk = plain.read(chunk_size)
+            if not chunk:
+                break
+            compressed.write(chunk.encode())
+    os.remove(source)
+    return destination
+
+
+def process_output(output, sanity_check=True, compress=True):
+    """Perform post-consistency-check actions.
+
+    DARK files are put in the quarantined-replica table so that they
+    may be deleted by the Dark Reaper.  LOST files are reported as
+    suspicious so that they may be further checked by the cloud squads.
+
+    ``output`` should be an ``str`` with the absolute path to the file
+    produced by ``consistency()``.  It must maintain its naming
+    convention.
+
+    If ``sanity_check`` is ``True`` (default) and the number of entries
+    in the output file is deemed excessive, the actions are aborted.
+
+    If ``compress`` is ``True`` (default), the file is compressed with
+    bzip2 after the actions are successfully performed.
+    """
+    logger = logging.getLogger('auditor-worker')
+    dark_replicas = []
+    lost_replicas = []
+    try:
+        with open(output) as f:
+            for line in f:
+                label, path = line.rstrip().split(',', 1)
+                scope, name = guess_replica_info(path)
+                if label == 'DARK':
+                    dark_replicas.append({'path': path,
+                                          'scope': InternalScope(scope),
+                                          'name': name})
+                elif label == 'LOST':
+                    lost_replicas.append({'scope': InternalScope(scope),
+                                          'name': name})
+                else:
+                    raise ValueError('unexpected label')
+    # Since the file is read immediately after its creation, any error
+    # exposes a bug in the Auditor.
+    except Exception as error:
+        logger.critical('Error processing "%s"', output, exc_info=True)
+        raise error
+
+    rse = os.path.basename(output[:output.rfind('_')])
+    rse_id = get_rse_id(rse=rse)
+    usage = get_rse_usage(rse_id=rse_id, source='rucio')[0]
+    threshold = config.config_get_float('auditor', 'threshold', False, 0.2)
+
+    # Perform a basic sanity check by comparing the number of entries
+    # with the total number of files on the RSE.  If the percentage is
+    # significant, there is most likely an issue with the site dump.
+    found_error = False
+    if len(dark_replicas) > threshold * usage['files']:
+        logger.warning('Number of DARK files is exceeding threshold: "%s"',
+                       output)
+        found_error = True
+    if len(lost_replicas) > threshold * usage['files']:
+        logger.warning('Number of LOST files is exceeding threshold: "%s"',
+                       output)
+        found_error = True
+    if found_error and sanity_check:
+        raise AssertionError('sanity check failed')
+
+    # While converting LOST replicas to PFNs, entries that do not
+    # correspond to a replica registered in Rucio are silently dropped.
+    lost_pfns = [r['rses'][rse_id][0] for chunk in chunks(lost_replicas, 1000) for r in list_replicas(chunk) if rse_id in r['rses']]
+
+    for chunk in chunks(dark_replicas, 1000):
+        add_quarantined_replicas(rse_id=rse_id, replicas=chunk)
+    logger.debug('Processed %d DARK files from "%s"', len(dark_replicas),
+                 output)
+    declare_bad_file_replicas(lost_pfns, reason='Reported by Auditor',
+                              issuer=InternalAccount('root'), status=BadFilesStatus.SUSPICIOUS)
+    logger.debug('Processed %d LOST files from "%s"', len(lost_replicas),
+                 output)
+
+    if compress:
+        destination = bz2_compress_file(output)
+        logger.debug('Compressed "%s"', destination)
+
+
+def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps, delta_in_days):
+    logger = logging.getLogger('auditor-worker')
+    lib_logger = logging.getLogger('dumper')
+
+    loglevel = logging.getLevelName(config.config_get('common', 'loglevel', False, 'DEBUG'))
+    logger.setLevel(loglevel)
+    lib_logger.setLevel(loglevel)
+
+    handler = LogPipeHandler(logpipe)
+    logger.addHandler(handler)
+    lib_logger.addHandler(handler)
+
+    formatter = logging.Formatter(
+        "%(asctime)s  %(name)-22s  %(levelname)-8s [PID %(process)8d] %(message)s"
+    )
+    handler.setFormatter(formatter)
+
+    delta = timedelta(days=delta_in_days)
+
+    configuration = srmdumps.parse_configuration()
+
+    while not terminate.is_set():
+        try:
+            rse, attemps = queue.get(timeout=30)
+        except Queue.Empty:
+            continue
+        start = datetime.now()
+        try:
+            logger.debug('Checking "%s"', rse)
+            output = consistency(rse, delta, configuration, cache_dir,
+                                 results_dir)
+            if output:
+                process_output(output)
+        except:
+            elapsed = (datetime.now() - start).total_seconds() / 60
+            logger.error('Check of "%s" failed in %d minutes, %d remaining attemps', rse, elapsed, attemps, exc_info=True)
+            success = False
+        else:
+            elapsed = (datetime.now() - start).total_seconds() / 60
+            logger.info('SUCCESS checking "%s" in %d minutes', rse, elapsed)
+            success = True
+
+        if not keep_dumps:
+            remove = glob.glob(os.path.join(cache_dir, 'replicafromhdfs_{0}_*'.format(rse)))
+            remove.extend(glob.glob(os.path.join(cache_dir, 'ddmendpoint_{0}_*'.format(rse))))
+            logger.debug('Removing: %s', remove)
+            for fil in remove:
+                os.remove(fil)
+
+        if not success and attemps > 0:
+            retry.put((rse, attemps - 1))
+
+
+def activity_logger(logpipes, logfilename, terminate):
+    handler = logging.handlers.RotatingFileHandler(
+        logfilename,
+        maxBytes=20971520,
+        backupCount=10,
+    )
+    handler.setFormatter(logging.Formatter(fmt=None))
+    logger = logging.getLogger('auditor-logger-raw')
+    logger.addHandler(handler)
+    logger.setLevel(logging.CRITICAL)  # The level of this logger is irrelevant
+
+    while not terminate.is_set():
+        ready, _, _ = select.select(logpipes, tuple(), tuple(), 30)
+        if ready:
+            for logpipe in ready:
+                logger.critical(logpipe.recv())

rucio/daemons/auditor/hdfs.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+
+from rucio.common.dumper import DUMPS_CACHE_DIR
+from rucio.common.dumper import temp_file
+from rucio.common.dumper.data_models import Replica
+
+
+def _hdfs_get(src_url, dst_path):
+    cmd = ['hadoop', 'fs', '-get', src_url, dst_path]
+    get = subprocess.Popen(
+        cmd,
+        stderr=subprocess.PIPE,
+    )
+    _, stderr = get.communicate()
+    if get.returncode != 0:
+        raise IOError('_hdfs_get(): "{0}": {1}. Return code {2}'.format(
+            ' '.join(cmd),
+            stderr,
+            get.returncode,
+        ))
+
+
+class ReplicaFromHDFS(Replica):
+    BASE_URL = '/user/rucio01/reports/{0}/replicas_per_rse/{1}/*'
+
+    @classmethod
+    def download(cls, rse, date, cache_dir=DUMPS_CACHE_DIR, buffer_size=65536):
+        logger = logging.getLogger('auditor.hdfs')
+
+        if not os.path.isdir(cache_dir):
+            os.mkdir(cache_dir)
+        tmp_dir = tempfile.mkdtemp(dir=cache_dir)
+
+        url = cls.BASE_URL.format(date.strftime('%Y-%m-%d'), rse)
+        filename = '{0}_{1}_{2}_{3}'.format(
+            cls.__name__.lower(),
+            rse,
+            date.strftime('%d-%m-%Y'),
+            hashlib.sha1(url.encode()).hexdigest()
+        )
+        filename = re.sub(r'\W', '-', filename)
+        path = os.path.join(cache_dir, filename)
+
+        if os.path.exists(path):
+            logger.debug('Taking Rucio Replica Dump %s for %s from cache', path, rse)
+            return path
+
+        try:
+            logging.debug('Trying to download: %s for %s', url, rse)
+
+            _hdfs_get(cls.BASE_URL.format(date.strftime('%Y-%m-%d'), rse), tmp_dir)
+            files = (os.path.join(tmp_dir, file_) for file_ in sorted(os.listdir(tmp_dir)))
+
+            with temp_file(cache_dir, filename, binary=True) as (full_dump, _):
+                for chunk_file in files:
+                    with open(chunk_file, 'rb') as partial_dump:
+                        while True:
+                            data_chunk = partial_dump.read(buffer_size)
+                            if not data_chunk:
+                                break
+                            full_dump.write(data_chunk)
+        finally:
+            shutil.rmtree(tmp_dir)
+
+        return path