rucio 32.8.6__py3-none-any.whl

rucio/daemons/badreplicas/necromancer.py

@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import re
+import threading
+import time
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING
+
+from dogpile.cache.api import NO_VALUE
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.cache import make_region_memcached
+from rucio.common.config import config_get_int
+from rucio.common.exception import DatabaseException
+from rucio.common.logging import setup_logging
+from rucio.core.monitor import MetricManager
+from rucio.core.replica import list_bad_replicas, get_replicas_state, get_bad_replicas_backlog
+from rucio.core.rule import (update_rules_for_lost_replica, update_rules_for_bad_replica,
+                             get_evaluation_backlog)
+from rucio.daemons.common import run_daemon
+from rucio.db.sqla.constants import ReplicaState
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.daemons.common import HeartbeatHandler
+
+graceful_stop = threading.Event()
+METRICS = MetricManager(module=__name__)
+REGION = make_region_memcached(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
+DAEMON_NAME = 'necromancer'
+
+
+def necromancer(bulk: int, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Creates a Necromancer Worker that gets a list of bad replicas for a given hash,
+    identify lost DIDs and for non-lost ones, set the locks and rules for reevaluation.
+
+    :param bulk: The number of requests to process.
+    :param once: Run only once.
+    :param sleep_time: Thread sleep time after each chunk of work.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    must_sleep = True
+
+    # Check if there is a Judge Evaluator backlog
+    max_evaluator_backlog_count = config_get_int('necromancer', 'max_evaluator_backlog_count', default=0, raise_exception=False)
+    max_evaluator_backlog_duration = config_get_int('necromancer', 'max_evaluator_backlog_duration', default=0, raise_exception=False)
+    backlog_refresh_time = config_get_int('necromancer', 'backlog_refresh_time', default=60, raise_exception=False)
+    if max_evaluator_backlog_count or max_evaluator_backlog_duration:
+        evaluator_backlog_count, evaluator_backlog_duration = get_evaluation_backlog(expiration_time=backlog_refresh_time)
+        if max_evaluator_backlog_count and \
+           evaluator_backlog_count and \
+           max_evaluator_backlog_duration and \
+           evaluator_backlog_duration and \
+           evaluator_backlog_count > max_evaluator_backlog_count and \
+           evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count and duration hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_count and evaluator_backlog_count and evaluator_backlog_count > max_evaluator_backlog_count:
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_duration and evaluator_backlog_duration and evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog duration hit, stopping operation')
+            return must_sleep
+
+    # Check how many bad replicas are queued
+    max_bad_replicas_backlog_count = config_get_int('necromancer', 'max_bad_replicas_backlog_count', default=0, raise_exception=False)
+    bad_replicas_backlog = REGION.get('bad_replicas_backlog')
+    if bad_replicas_backlog is NO_VALUE:
+        bad_replicas_backlog = get_bad_replicas_backlog()
+        REGION.set('bad_replicas_backlog', bad_replicas_backlog)
+    tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
+    list_of_rses = list()
+    # If too many replica, call list_bad_replicas with a list of RSEs
+    if max_bad_replicas_backlog_count and tot_bad_files > max_bad_replicas_backlog_count and len(bad_replicas_backlog) > 1:
+        logger(logging.INFO, 'Backlog of bads replica too big. Apply some sharing between different RSEs')
+        rses = list()
+        cnt = 0
+        for key in sorted(bad_replicas_backlog, key=bad_replicas_backlog.get, reverse=True):
+            rses.append({'id': key})
+            cnt += bad_replicas_backlog[key]
+            if cnt >= bulk:
+                list_of_rses.append(rses)
+                rses = list()
+                cnt = 0
+    else:
+        list_of_rses.append(None)
+
+    tot_processed = 0
+    if tot_bad_files == 0:
+        logger(logging.INFO, 'No bad replicas to process.')
+    else:
+        ttime = time.time()
+        replicas = []
+        for rses in list_of_rses:
+
+            worker_number, total_workers, logger = heartbeat_handler.live()
+            replicas = list_bad_replicas(limit=bulk, thread=worker_number, total_threads=total_workers, rses=rses)
+            for replica in replicas:
+                scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
+                logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
+
+                list_replicas = get_replicas_state(scope=scope, name=name)
+                if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
+                    logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
+                    try:
+                        update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        METRICS.counter(name='badfiles.lostfile').inc()
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                        else:
+                            logger(logging.ERROR, str(error))
+
+                else:
+                    rep = list_replicas.get(ReplicaState.AVAILABLE, [])
+                    unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
+                    logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
+                    try:
+                        update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        METRICS.counter(name='badfiles.recovering').inc()
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                        else:
+                            logger(logging.ERROR, str(error))
+
+            tot_processed += len(replicas)
+            logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - ttime), str(len(replicas))))
+
+    if tot_processed == 0 or tot_bad_files == 0:
+        return must_sleep
+    must_sleep = False
+    return must_sleep
+
+
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Starts up the necromancer threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('Will run only one iteration in a single threaded mode')
+        necromancer(bulk=bulk, once=once)
+    else:
+        logging.info('starting necromancer threads')
+        thread_list = [threading.Thread(target=necromancer, kwargs={'once': once,
+                                                                    'bulk': bulk,
+                                                                    'sleep_time': sleep_time}) for _ in range(0, threads)]
+        [t.start() for t in thread_list]
+
+        logging.info('waiting for interrupts')
+
+        # Interruptible joins require a timeout.
+        while thread_list:
+            thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()

rucio/daemons/bb8/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/bb8/bb8.py

@@ -0,0 +1,356 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+BB8 is a daemon the re-balance data between RSEs.
+"""
+
+import functools
+import logging
+import socket
+import threading
+from typing import TYPE_CHECKING
+
+from rucio.common.config import config_get_float
+from rucio.common.exception import InvalidRSEExpression
+from rucio.common.logging import setup_logging
+from rucio.core.heartbeat import sanity_check, list_payload_counts
+from rucio.core.rse import get_rse_usage
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.bb8.common import rebalance_rse, get_active_locks
+from rucio.daemons.common import run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.daemons.common import HeartbeatHandler
+
+graceful_stop = threading.Event()
+DAEMON_NAME = "rucio-bb8"
+
+
+def rule_rebalancer(
+    rse_expression: str,
+    move_subscriptions: bool = False,
+    use_dump: bool = False,
+    sleep_time: int = 300,
+    once: bool = True,
+    dry_run: bool = False,
+) -> None:
+    """
+    Create a rule_rebalancer worker
+
+    :param rse_expression: The RSE expression where the rule rebalancing is applied.
+    :param move_subscription: To allow rebalancing of subscription rules. Not implemented yet.
+    :param use_dump: To use dump instead of DB query.
+    :param sleep_time: Time between two cycles.
+    :param once: Run only once.
+    :param dry_run: To run in dry run mode (i.e. rules are not created).
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            rse_expression=rse_expression,
+            move_subscriptions=move_subscriptions,
+            use_dump=use_dump,
+            dry_run=dry_run,
+        ),
+    )
+
+
+def run_once(
+    heartbeat_handler: "HeartbeatHandler",
+    rse_expression: str,
+    move_subscriptions: bool,
+    use_dump: bool,
+    dry_run: bool,
+    **_kwargs
+) -> bool:
+
+    must_sleep = False
+    total_rebalance_volume = 0
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.DEBUG, "Running BB8 on rse_expression: %s", rse_expression)
+    tolerance = config_get_float("bb8", "tolerance", default=0.05)
+    max_total_rebalance_volume = config_get_float(
+        "bb8", "max_total_rebalance_volume", default=10 * 1e12
+    )
+    max_rse_rebalance_volume = config_get_float(
+        "bb8", "max_rse_rebalance_volume", default=500 * 1e9
+    )
+    min_total = config_get_float("bb8", "min_total", default=20 * 1e9)
+    payload_cnt = list_payload_counts(
+        executable=DAEMON_NAME, older_than=600, hash_executable=None, session=None
+    )
+    if rse_expression in payload_cnt:
+        logger(
+            logging.WARNING,
+            "One BB8 instance already running with the same RSE expression. Stopping",
+        )
+        must_sleep = True
+        return must_sleep
+    else:
+        # List the RSEs represented by rse_expression
+        try:
+            rses = [rse for rse in parse_expression(rse_expression)]
+            list_rses2 = [rse["rse"] for rse in rses]
+        except InvalidRSEExpression as err:
+            logger(logging.ERROR, err)
+            return must_sleep
+        # List the RSEs represented by all the RSE expressions stored in heartbeat payload
+        list_rses1 = []
+        for rse_exp in payload_cnt:
+            if rse_exp:
+                list_rses1 = [rse["rse"] for rse in parse_expression(rse_exp)]
+        for rse in list_rses2:
+            if rse in list_rses1:
+                logger(
+                    logging.WARNING,
+                    "Overlapping RSE expressions %s vs %s. Stopping",
+                    rse_exp,
+                    rse_expression,
+                )
+                return must_sleep
+
+        logger(logging.INFO, "Will process rebalancing on %s", rse_expression)
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        total_primary = 0
+        total_secondary = 0
+        total_total = 0
+        global_ratio = float(0)
+        for rse in rses:
+            logger(logging.DEBUG, "Getting RSE usage on %s", rse["rse"])
+            rse_usage = get_rse_usage(rse_id=rse["id"])
+            usage_dict = {}
+            for item in rse_usage:
+                # TODO Check last update
+                usage_dict[item["source"]] = {
+                    "used": item["used"],
+                    "free": item["free"],
+                    "total": item["total"],
+                }
+
+            try:
+                rse["primary"] = (
+                    usage_dict["rucio"]["used"] - usage_dict["expired"]["used"]
+                )
+                rse["secondary"] = usage_dict["expired"]["used"]
+                rse["total"] = (
+                    usage_dict["storage"]["total"]
+                    - usage_dict["min_free_space"]["used"]
+                )
+                rse["ratio"] = float(rse["primary"]) / float(rse["total"])
+            except KeyError as err:
+                logger(
+                    logging.ERROR,
+                    "Missing source usage %s for RSE %s. Exiting",
+                    err,
+                    rse["rse"],
+                )
+                break
+            total_primary += rse["primary"]
+            total_secondary += rse["secondary"]
+            total_total += float(rse["total"])
+            rse["receive_volume"] = 0  # Already rebalanced volume in this run
+            global_ratio = float(total_primary) / float(total_total)
+            logger(logging.INFO, "Global ratio: %f" % (global_ratio))
+
+        for rse in sorted(rses, key=lambda k: k["ratio"]):
+            logger(
+                logging.INFO,
+                "%s Sec/Prim local ratio (%f) vs global %s",
+                rse["rse"],
+                rse["ratio"],
+                global_ratio,
+            )
+        rses_over_ratio = sorted(
+            [
+                rse
+                for rse in rses
+                if rse["ratio"] > global_ratio + global_ratio * tolerance
+            ],
+            key=lambda k: k["ratio"],
+            reverse=True,
+        )
+        rses_under_ratio = sorted(
+            [
+                rse
+                for rse in rses
+                if rse["ratio"] < global_ratio - global_ratio * tolerance
+            ],
+            key=lambda k: k["ratio"],
+            reverse=False,
+        )
+
+        # Excluding RSEs
+        logger(
+            logging.DEBUG, "Excluding RSEs as destination which are too small by size:"
+        )
+        for des in rses_under_ratio:
+            if des["total"] < min_total:
+                logger(logging.DEBUG, "Excluding %s", des["rse"])
+                rses_under_ratio.remove(des)
+        logger(logging.DEBUG, "Excluding RSEs as sources which are too small by size:")
+        for src in rses_over_ratio:
+            if src["total"] < min_total:
+                logger(logging.DEBUG, "Excluding %s", src["rse"])
+                rses_over_ratio.remove(src)
+        logger(
+            logging.DEBUG,
+            "Excluding RSEs as destinations which are not available for write:",
+        )
+        for des in rses_under_ratio:
+            if not des["availability_write"]:
+                logger(logging.DEBUG, "Excluding %s", des["rse"])
+                rses_under_ratio.remove(des)
+        logger(
+            logging.DEBUG, "Excluding RSEs as sources which are not available for read:"
+        )
+        for src in rses_over_ratio:
+            if not src["availability_read"]:
+                logger(logging.DEBUG, "Excluding %s", src["rse"])
+                rses_over_ratio.remove(src)
+
+        # Gets the number of active transfers per location
+        dict_locks = get_active_locks(session=None)
+
+        # Loop over RSEs over the ratio
+        for index, source_rse in enumerate(rses_over_ratio):
+
+            # The volume that would be rebalanced, not real availability of the data:
+            available_source_rebalance_volume = int(
+                (source_rse["primary"] - global_ratio * source_rse["secondary"])
+                / (global_ratio + 1)
+            )
+            if available_source_rebalance_volume > max_rse_rebalance_volume:
+                available_source_rebalance_volume = max_rse_rebalance_volume
+            if (
+                available_source_rebalance_volume
+                > max_total_rebalance_volume - total_rebalance_volume
+            ):
+                available_source_rebalance_volume = (
+                    max_total_rebalance_volume - total_rebalance_volume
+                )
+
+            # Select a target:
+            for destination_rse in rses_under_ratio:
+                if available_source_rebalance_volume > 0:
+                    vo_str = (
+                        " on VO {}".format(destination_rse["vo"])
+                        if destination_rse["vo"] != "def"
+                        else ""
+                    )
+                    if index == 0 and destination_rse["id"] in dict_locks:
+                        replicating_volume = dict_locks[destination_rse["id"]]["bytes"]
+                        logger(
+                            logging.DEBUG,
+                            "Already %f TB replicating to %s%s",
+                            replicating_volume / 1e12,
+                            destination_rse["rse"],
+                            vo_str,
+                        )
+                        destination_rse["receive_volume"] += replicating_volume
+                    if destination_rse["receive_volume"] >= max_rse_rebalance_volume:
+                        continue
+                    available_target_rebalance_volume = (
+                        max_rse_rebalance_volume - destination_rse["receive_volume"]
+                    )
+                    if (
+                        available_target_rebalance_volume
+                        >= available_source_rebalance_volume
+                    ):
+                        available_target_rebalance_volume = (
+                            available_source_rebalance_volume
+                        )
+
+                    logger(
+                        logging.INFO,
+                        "Rebalance %d TB from %s(%f) to %s(%f)%s",
+                        available_target_rebalance_volume / 1e12,
+                        source_rse["rse"],
+                        source_rse["ratio"],
+                        destination_rse["rse"],
+                        destination_rse["ratio"],
+                        vo_str,
+                    )
+                    expr = destination_rse["rse"]
+                    rebalance_rse(
+                        rse_id=source_rse["id"],
+                        max_bytes=available_target_rebalance_volume,
+                        dry_run=dry_run,
+                        comment="Background rebalancing",
+                        force_expression=expr,
+                        logger=logger,
+                    )
+
+                    destination_rse[
+                        "receive_volume"
+                    ] += available_target_rebalance_volume
+                    total_rebalance_volume += available_target_rebalance_volume
+                    available_source_rebalance_volume -= (
+                        available_target_rebalance_volume
+                    )
+
+    must_sleep = True
+    return must_sleep
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(
+    once: bool,
+    rse_expression: str,
+    move_subscriptions: bool = False,
+    use_dump: bool = False,
+    sleep_time: int = 300,
+    threads: int = 1,
+    dry_run: bool = False,
+) -> None:
+    """
+    Starts up the BB8 rebalancing threads.
+    """
+
+    setup_logging(process_name=DAEMON_NAME)
+    hostname = socket.gethostname()
+    sanity_check(executable=DAEMON_NAME, hostname=hostname)
+    logging.info("BB8 starting %s threads", str(threads))
+    threads = [
+        threading.Thread(
+            target=rule_rebalancer,
+            kwargs={
+                "once": once,
+                "rse_expression": rse_expression,
+                "sleep_time": sleep_time,
+                "dry_run": dry_run,
+            },
+        )
+        for _ in range(0, threads)
+    ]
+    [thread.start() for thread in threads]
+    # Interruptible joins require a timeout.
+    while threads[0].is_alive():
+        [thread.join(timeout=3.14) for thread in threads]