rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/badreplicas/minos_temporary_expiration.py

@@ -0,0 +1,171 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import math
+import re
+import threading
+import traceback
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound
+from rucio.common.logging import setup_logging
+from rucio.common.utils import chunks
+from rucio.core.did import get_metadata
+from rucio.core.replica import bulk_delete_bad_replicas, get_replicas_state, list_expired_temporary_unavailable_replicas, update_replicas_states
+from rucio.daemons.common import run_daemon
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, ReplicaState
+from rucio.db.sqla.session import get_session
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.daemons.common import HeartbeatHandler
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'minos-temporary-expiration'
+
+
+def minos_tu_expiration(bulk: int = 1000, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Creates a Minos Temporary Unavailable Replicas Expiration Worker that
+    gets the list of expired TU replicas and sets them back to AVAILABLE.
+
+    :param bulk: The number of requests to process.
+    :param once: Run only once.
+    :param sleep_time: Time between two cycles.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    chunk_size = 10  # The chunk size used for the commits
+    logger(logging.INFO, 'Getting list of expired replicas')
+    expired_replicas = list_expired_temporary_unavailable_replicas(total_workers=total_workers,
+                                                                   worker_number=worker_number,
+                                                                   limit=1000)
+    logger(logging.INFO, '%s expired replicas returned', len(expired_replicas))
+    logger(logging.DEBUG, 'List of expired replicas returned %s', str(expired_replicas))
+    replicas = []
+    bad_replicas = []
+    nchunk = 0
+    tot_chunk = int(math.ceil(len(expired_replicas) / float(chunk_size)))
+    session = get_session()
+    for chunk in chunks(expired_replicas, chunk_size):
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        skip_replica_update = []
+        # Process and update the replicas in chunks
+        for replica in chunk:
+            scope, name, rse_id = replica[0], replica[1], replica[2]
+            states_dictionary = get_replicas_state(scope=scope, name=name, session=session)
+            # Check if the replica is not declared bad
+            # If already declared bad don't update the replica state, but remove from bad_pfns
+            if not (ReplicaState.BAD in states_dictionary and rse_id in states_dictionary[ReplicaState.BAD]):
+                replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE})
+            else:
+                skip_replica_update.append((scope, name))
+            # Remove the replicas from bad_replicas table in chunks
+            bad_replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE})
+        try:
+            nchunk += 1
+            logger(logging.DEBUG, 'Running on %s chunk out of %s', nchunk, tot_chunk)
+            update_replicas_states(replicas, nowait=True, session=session)
+            bulk_delete_bad_replicas(bad_replicas, session=session)
+            session.commit()  # pylint: disable=no-member
+        except (ReplicaNotFound, DataIdentifierNotFound) as error:
+            session.rollback()  # pylint: disable=no-member
+            logger(logging.WARNING, 'One of the replicas does not exist anymore. Updating and deleting one by one. Error : %s', str(error))
+            for replica in chunk:
+                _, _, logger = heartbeat_handler.live()
+                scope, name, rse_id = replica[0], replica[1], replica[2]
+                logger(logging.DEBUG, 'Working on %s:%s on %s', scope, name, rse_id)
+                try:
+                    # First check if the DID exists
+                    get_metadata(scope, name)
+                    if (scope, name) not in skip_replica_update:
+                        update_replicas_states([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE}, ], nowait=True, session=session)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
+                    session.commit()  # pylint: disable=no-member
+                except DataIdentifierNotFound:
+                    session.rollback()  # pylint: disable=no-member
+                    logger(logging.WARNING, 'DID %s:%s does not exist anymore.', scope, name)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
+                    session.commit()  # pylint: disable=no-member
+                except ReplicaNotFound:
+                    session.rollback()  # pylint: disable=no-member
+                    logger(logging.WARNING, 'Replica %s:%s on RSEID %s does not exist anymore.', scope, name, rse_id)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
+                    session.commit()  # pylint: disable=no-member
+            session = get_session()
+        except (DatabaseException, DatabaseError) as error:
+            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+            else:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+            session.rollback()
+            session = get_session()
+        except Exception:
+            session.rollback()  # pylint: disable=no-member
+            logger(logging.CRITICAL, str(traceback.format_exc()))
+            session = get_session()
+    return True
+
+
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Starts up the minos threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.log(logging.INFO, 'Will run only one iteration in a single threaded mode')
+        minos_tu_expiration(bulk=bulk, once=once)
+    else:
+        logging.log(logging.INFO, 'Starting Minos Temporary Expiration threads')
+        thread_list = [threading.Thread(target=minos_tu_expiration, kwargs={'once': once,
+                                                                            'sleep_time': sleep_time,
+                                                                            'bulk': bulk}) for _ in range(0, threads)]
+        [thread.start() for thread in thread_list]
+        logging.log(logging.INFO, 'Waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while thread_list:
+            thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()

rucio/daemons/badreplicas/necromancer.py

@@ -0,0 +1,196 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import re
+import threading
+import time
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Optional
+
+from dogpile.cache.api import NO_VALUE
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.cache import MemcacheRegion
+from rucio.common.config import config_get_int
+from rucio.common.exception import DatabaseException
+from rucio.common.logging import setup_logging
+from rucio.core.monitor import MetricManager
+from rucio.core.replica import get_bad_replicas_backlog, get_replicas_state, list_bad_replicas
+from rucio.core.rule import get_evaluation_backlog, update_rules_for_bad_replica, update_rules_for_lost_replica
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, ReplicaState
+
+if TYPE_CHECKING:
+    from types import FrameType
+
+
+graceful_stop = threading.Event()
+METRICS = MetricManager(module=__name__)
+REGION = MemcacheRegion(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
+DAEMON_NAME = 'necromancer'
+
+
+def necromancer(bulk: int, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Creates a Necromancer Worker that gets a list of bad replicas for a given hash,
+    identify lost DIDs and for non-lost ones, set the locks and rules for reevaluation.
+
+    :param bulk: The number of requests to process.
+    :param once: Run only once.
+    :param sleep_time: Thread sleep time after each chunk of work.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: HeartbeatHandler, bulk: int, **_kwargs) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    must_sleep = True
+
+    # Check if there is a Judge Evaluator backlog
+    max_evaluator_backlog_count = config_get_int('necromancer', 'max_evaluator_backlog_count', default=0, raise_exception=False)
+    max_evaluator_backlog_duration = config_get_int('necromancer', 'max_evaluator_backlog_duration', default=0, raise_exception=False)
+    backlog_refresh_time = config_get_int('necromancer', 'backlog_refresh_time', default=60, raise_exception=False)
+    if max_evaluator_backlog_count or max_evaluator_backlog_duration:
+        evaluator_backlog_count, evaluator_backlog_duration = get_evaluation_backlog(expiration_time=backlog_refresh_time)
+        if max_evaluator_backlog_count and \
+           evaluator_backlog_count and \
+           max_evaluator_backlog_duration and \
+           evaluator_backlog_duration and \
+           evaluator_backlog_count > max_evaluator_backlog_count and \
+           evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count and duration hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_count and evaluator_backlog_count and evaluator_backlog_count > max_evaluator_backlog_count:
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_duration and evaluator_backlog_duration and evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog duration hit, stopping operation')
+            return must_sleep
+
+    # Check how many bad replicas are queued
+    max_bad_replicas_backlog_count = config_get_int('necromancer', 'max_bad_replicas_backlog_count', default=0, raise_exception=False)
+    bad_replicas_backlog = REGION.get('bad_replicas_backlog')
+    if bad_replicas_backlog is NO_VALUE:
+        bad_replicas_backlog = get_bad_replicas_backlog()
+        REGION.set('bad_replicas_backlog', bad_replicas_backlog)
+    tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
+    list_of_rses = list()
+    # If too many replica, call list_bad_replicas with a list of RSEs
+    if max_bad_replicas_backlog_count and tot_bad_files > max_bad_replicas_backlog_count and len(bad_replicas_backlog) > 1:
+        logger(logging.INFO, 'Backlog of bads replica too big. Apply some sharing between different RSEs')
+        rses = list()
+        cnt = 0
+        for key in sorted(bad_replicas_backlog, key=bad_replicas_backlog.get, reverse=True):
+            rses.append({'id': key})
+            cnt += bad_replicas_backlog[key]
+            if cnt >= bulk:
+                list_of_rses.append(rses)
+                rses = list()
+                cnt = 0
+    else:
+        list_of_rses.append(None)
+
+    tot_processed = 0
+    if tot_bad_files == 0:
+        logger(logging.INFO, 'No bad replicas to process.')
+    else:
+        ttime = time.time()
+        replicas = []
+        for rses in list_of_rses:
+
+            worker_number, total_workers, logger = heartbeat_handler.live()
+            replicas = list_bad_replicas(limit=bulk, thread=worker_number, total_threads=total_workers, rses=rses)
+            for replica in replicas:
+                scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
+                logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
+
+                list_replicas = get_replicas_state(scope=scope, name=name)
+                if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
+                    logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
+                    try:
+                        update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        METRICS.counter(name='badfiles.lostfile').inc()
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                        else:
+                            logger(logging.ERROR, str(error))
+
+                else:
+                    rep = list_replicas.get(ReplicaState.AVAILABLE, [])
+                    unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
+                    logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
+                    try:
+                        update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        METRICS.counter(name='badfiles.recovering').inc()
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                        else:
+                            logger(logging.ERROR, str(error))
+
+            tot_processed += len(replicas)
+            logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - ttime), str(len(replicas))))
+
+    if tot_processed == 0 or tot_bad_files == 0:
+        return must_sleep
+    must_sleep = False
+    return must_sleep
+
+
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Starts up the necromancer threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('Will run only one iteration in a single threaded mode')
+        necromancer(bulk=bulk, once=once)
+    else:
+        logging.info('starting necromancer threads')
+        thread_list = [threading.Thread(target=necromancer, kwargs={'once': once,
+                                                                    'bulk': bulk,
+                                                                    'sleep_time': sleep_time}) for _ in range(0, threads)]
+        [t.start() for t in thread_list]
+
+        logging.info('waiting for interrupts')
+
+        # Interruptible joins require a timeout.
+        while thread_list:
+            thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
+
+
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()

rucio/daemons/bb8/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.