rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/tracer/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/tracer/kronos.py

@@ -0,0 +1,511 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This daemon consumes tracer messages from ActiveMQ and updates the atime for replicas.
+"""
+import functools
+import logging
+import re
+from configparser import NoOptionError, NoSectionError
+from datetime import datetime
+from json import dumps as jdumps
+from json import loads as jloads
+from queue import Queue
+from threading import Event, Thread
+from time import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common.config import config_get, config_get_int, config_get_list
+from rucio.common.exception import DatabaseException, RSENotFound
+from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.stomp_utils import ListenerBase, StompConnectionManager
+from rucio.common.stopwatch import Stopwatch
+from rucio.common.types import InternalAccount, InternalScope, LoggerFunction
+from rucio.core.did import list_parent_dids, touch_dids
+from rucio.core.lock import touch_dataset_locks
+from rucio.core.monitor import MetricManager
+from rucio.core.replica import declare_bad_file_replicas, touch_collection_replicas, touch_replica
+from rucio.core.rse import get_rse_id
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+from rucio.db.sqla.constants import BadFilesStatus, DIDType
+
+if TYPE_CHECKING:
+    from collections.abc import Set
+    from types import FrameType
+
+    from stomp.utils import Frame
+
+    from rucio.common.stomp_utils import Connection
+
+
+logging.getLogger("stomp").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+graceful_stop = Event()
+
+
+class AMQConsumer(ListenerBase):
+    """ActiveMQ message consumer"""
+
+    def __init__(self,
+                 conn: "Connection",
+                 queue: str,
+                 chunksize: int,
+                 subscription_id: str,
+                 excluded_usrdns: "Set[str]",
+                 dataset_queue: Queue,
+                 bad_files_patterns: list[re.Pattern],
+                 logger: LoggerFunction = logging.log,
+                 **kwargs: dict) -> None:
+        super().__init__(conn=conn, logger=logger, **kwargs)
+        self.__queue = queue
+        self.__reports = []
+        self.__ids = []
+        self.__chunksize = chunksize
+        self.__subscription_id = subscription_id
+        # excluded states empty for the moment, maybe that should be recosidered in the future
+        self.__excluded_states = set([])
+        # exclude specific usrdns like GangaRBT
+        self.__excluded_usrdns = excluded_usrdns
+        self.__dataset_queue = dataset_queue
+        self.__bad_files_patterns = bad_files_patterns
+
+    @METRICS.count_it
+    def on_message(self, frame: "Frame") -> None:
+        appversion = 'dq2'
+        msg_id = frame.headers['message-id']
+        if 'appversion' in frame.headers:
+            appversion = frame.headers['appversion']
+
+        if 'resubmitted' in frame.headers:
+            METRICS.counter('received_resubmitted').inc()
+
+        try:
+            if appversion == 'dq2':
+                self._conn.ack(msg_id, self.__subscription_id)
+                return
+            else:
+                report = jloads(frame.body)  # type: ignore
+        except Exception:
+            # message is corrupt, not much to do here
+            # send count to graphite, send ack to broker and return
+            METRICS.counter('json_error').inc()
+            self._logger(logging.ERROR, 'json error', exc_info=True)
+            self._conn.ack(msg_id, self.__subscription_id)
+            return
+
+        self.__ids.append(msg_id)
+        self.__reports.append(report)
+
+        try:
+            self._logger(logging.DEBUG,
+                         "message received: %s %s %s",
+                         str(report['eventType']),
+                         report['filename'],
+                         report['remoteSite'])
+        except Exception:
+            pass
+
+        if len(self.__ids) >= self.__chunksize:
+            self.__update_atime()
+            for msg_id in self.__ids:
+                self._conn.ack(msg_id, self.__subscription_id)
+
+            self.__reports = []
+            self.__ids = []
+
+    def __update_atime(self) -> None:
+        """
+        Bulk update atime.
+        """
+        replicas = []
+        rses = []
+        for report in self.__reports:
+            if 'vo' not in report:
+                report['vo'] = 'def'
+
+            try:
+                # Identify suspicious files
+                try:
+                    if self.__bad_files_patterns and report['eventType'] in ['get_sm', 'get_sm_a', 'get'] and 'clientState' in report and report['clientState'] not in ['DONE', 'FOUND_ROOT', 'ALREADY_DONE']:
+                        for pattern in self.__bad_files_patterns:
+                            if 'stateReason' in report and report['stateReason'] and isinstance(report['stateReason'], str) and pattern.match(report['stateReason']):
+                                reason = report['stateReason'][:255]
+                                if 'url' not in report or not report['url']:
+                                    self._logger(logging.ERROR, 'Missing url in the following trace : ' + str(report))
+                                else:
+                                    try:
+                                        surl = report['url']
+                                        declare_bad_file_replicas([surl, ], reason=reason, issuer=InternalAccount('root', vo=report['vo']), status=BadFilesStatus.SUSPICIOUS)
+                                        self._logger(logging.INFO,
+                                                     "Declare suspicious file %s with reason %s",
+                                                     report['url'],
+                                                     reason)
+                                    except Exception as error:
+                                        self._logger(logging.ERROR, 'Failed to declare suspicious file' + str(error))
+                except Exception as error:
+                    self._logger(logging.ERROR, 'Problem with bad trace : %s . Error %s', str(report), str(error))
+
+                # check if scope in report. if not skip this one.
+                if 'scope' not in report:
+                    METRICS.counter('missing_scope').inc()
+                    if report['eventType'] != 'touch':
+                        continue
+                else:
+                    METRICS.counter('with_scope').inc()
+                    report['scope'] = InternalScope(report['scope'], report['vo'])
+
+                # handle all events starting with get* and download and touch events.
+                if not report['eventType'].startswith('get') and not report['eventType'].startswith('sm_get') and not report['eventType'] == 'download' and not report['eventType'] == 'touch':
+                    continue
+                if report['eventType'].endswith('_es'):
+                    continue
+                METRICS.counter('total_get').inc()
+                if report['eventType'] == 'get':
+                    METRICS.counter('dq2clients').inc()
+                elif report['eventType'] == 'get_sm' or report['eventType'] == 'sm_get':
+                    if report['eventVersion'] == 'aCT':
+                        METRICS.counter('panda_production_act').inc()
+                    else:
+                        METRICS.counter('panda_production').inc()
+                elif report['eventType'] == 'get_sm_a' or report['eventType'] == 'sm_get_a':
+                    if report['eventVersion'] == 'aCT':
+                        METRICS.counter('panda_analysis_act').inc()
+                    else:
+                        METRICS.counter('panda_analysis').inc()
+                elif report['eventType'] == 'download':
+                    METRICS.counter('rucio_download').inc()
+                elif report['eventType'] == 'touch':
+                    METRICS.counter('rucio_touch').inc()
+                else:
+                    METRICS.counter('other_get').inc()
+
+                if report['eventType'] == 'download' or report['eventType'] == 'touch':
+                    report['usrdn'] = report['account']
+
+                if report['usrdn'] in self.__excluded_usrdns:
+                    continue
+                # handle touch and non-touch traces differently
+                if report['eventType'] != 'touch':
+                    # check if the report has the right state.
+                    if 'eventVersion' in report:
+                        if report['eventVersion'] != 'aCT':
+                            if report['clientState'] in self.__excluded_states:
+                                continue
+
+                    if 'remoteSite' not in report:
+                        continue
+                    if not report['remoteSite']:
+                        continue
+
+                    if 'filename' not in report:
+                        if 'name' in report:
+                            report['filename'] = report['name']
+
+                    rses = report['remoteSite'].strip().split(',')
+                    for rse in rses:
+                        try:
+                            rse_id = get_rse_id(rse=rse, vo=report['vo'])
+                        except RSENotFound:
+                            self._logger(logging.WARNING, "Cannot lookup rse_id for %s. Will skip this report.", rse)
+                            METRICS.counter('rse_not_found').inc()
+                            continue
+                        replicas.append({'name': report['filename'], 'scope': report['scope'], 'rse': rse, 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix']),
+                                         'traceTimeentryUnix': report['traceTimeentryUnix'], 'eventVersion': report['eventVersion']})
+                else:
+                    # if touch event and if datasetScope is in the report then it means
+                    # that there is no file scope/name and therefore only the dataset is
+                    # put in the queue to be updated and the rest is skipped.
+                    rse_id = None
+                    rse = None
+                    if 'remoteSite' in report:
+                        rse = report['remoteSite']
+                        try:
+                            rse_id = get_rse_id(rse=rse, vo=report['vo'])
+                        except RSENotFound:
+                            self._logger(logging.WARNING, "Cannot lookup rse_id for %s.", rse)
+                            METRICS.counter('rse_not_found').inc()
+                    if 'datasetScope' in report:
+                        self.__dataset_queue.put({'scope': InternalScope(report['datasetScope'], vo=report['vo']),
+                                                  'name': report['dataset'],
+                                                  'rse_id': rse_id,
+                                                  'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
+                        continue
+                    else:
+                        if 'remoteSite' not in report:
+                            continue
+                        replicas.append({'name': report['filename'],
+                                         'scope': report['scope'],
+                                         'rse': rse,
+                                         'rse_id': rse_id,
+                                         'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
+
+            except (KeyError, AttributeError):
+                self._logger(logging.ERROR, "Cannot handle report.", exc_info=True)
+                METRICS.counter('report_error').inc()
+                continue
+            except Exception:
+                self._logger(logging.ERROR, "Exception", exc_info=True)
+                continue
+
+            for did in list_parent_dids(report['scope'], report['filename']):
+                if did['type'] != DIDType.DATASET:
+                    continue
+                # do not update _dis datasets
+                if did['scope'].external == 'panda' and '_dis' in did['name']:
+                    continue
+                for rse in rses:
+                    try:
+                        rse_id = get_rse_id(rse=rse, vo=report['vo'])
+                    except RSENotFound:
+                        self._logger(logging.WARNING, "Cannot lookup rse_id for %s. Will skip this report.", rse)
+                        METRICS.counter('rse_not_found').inc()
+                        continue
+                    self.__dataset_queue.put({'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
+
+        if not replicas:
+            return
+
+        self._logger(logging.DEBUG, "trying to update replicas: %s", replicas)
+
+        stopwatch = Stopwatch()
+        try:
+            for replica in replicas:
+                # if touch replica hits a locked row put the trace back into queue for later retry
+                if not touch_replica(replica):
+                    resubmit = {'filename': replica['name'],
+                                'scope': replica['scope'].external,
+                                'remoteSite': replica['rse'],
+                                'traceTimeentryUnix': replica['traceTimeentryUnix'],
+                                'eventType': 'get',
+                                'usrdn': 'someuser',
+                                'clientState': 'DONE',
+                                'eventVersion': replica['eventVersion']}
+                    if replica['scope'].vo != 'def':
+                        resubmit['vo'] = replica['scope'].vo
+                    self._conn.send(body=jdumps(resubmit), destination=self.__queue, headers={'appversion': 'rucio', 'resubmitted': '1'})
+                    METRICS.counter('sent_resubmitted').inc()
+            METRICS.timer('update_atime').observe(stopwatch.elapsed)
+        except Exception:
+            self._logger(logging.ERROR, "Cannot update replicas.", exc_info=True)
+            METRICS.counter('update_error').inc()
+
+        METRICS.counter('updated_replicas').inc()
+
+
+def kronos_file(once: bool = False,
+                dataset_queue: "Queue | None" = None,
+                sleep_time: int = 60,
+                logger: LoggerFunction = logging.log) -> None:
+    """
+    Main loop to consume tracer reports.
+    """
+    bad_files_patterns = []
+    try:
+        patterns = config_get_list(section='kronos', option='bad_files_patterns', session=None)
+        for pat in patterns:
+            bad_files_patterns.append(re.compile(pat.strip()))
+    except (NoOptionError, NoSectionError, RuntimeError):
+        logger(logging.ERROR, "Failed to get bad_file_patterns")
+        bad_files_patterns.clear()
+    except Exception as error:
+        logger(logging.ERROR, f'Failed to get bad_file_patterns {str(error)}')
+        bad_files_patterns.clear()
+
+    excluded_usrdns = set(config_get('tracer-kronos', 'excluded_usrdns').split(','))
+
+    subscription_id = config_get('tracer-kronos', 'subscription_id')
+    stomp_conn_mngr = StompConnectionManager(config_section='tracer-kronos',
+                                             logger=logger)
+    stomp_conn_mngr.set_listener_factory('rucio-tracer-kronos',
+                                         AMQConsumer,
+                                         queue=config_get('tracer-kronos', 'queue'),
+                                         chunksize=config_get_int('tracer-kronos', 'chunksize'),
+                                         subscription_id=subscription_id,
+                                         excluded_usrdns=excluded_usrdns,
+                                         dataset_queue=dataset_queue,
+                                         bad_files_patterns=bad_files_patterns,
+                                         heartbeats=stomp_conn_mngr.config.heartbeats)
+
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable='kronos-file',
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once_kronos_file,
+            stomp_conn_mngr=stomp_conn_mngr,
+            dataset_queue=dataset_queue,  # type: ignore
+            sleep_time=sleep_time,
+        )
+    )
+    stomp_conn_mngr.disconnect()
+
+
+def run_once_kronos_file(heartbeat_handler: HeartbeatHandler,
+                         stomp_conn_mngr: StompConnectionManager,
+                         dataset_queue: Queue,
+                         sleep_time: int,
+                         **kwargs: dict) -> None:
+    """
+    Run the amq consumer once.
+    """
+    _, _, logger = heartbeat_handler.live()
+
+    stomp_conn_mngr.subscribe(id_=config_get('tracer-kronos', 'subscription_id'),
+                              ack='client-individual',
+                              destination=config_get('tracer-kronos', 'queue'),
+                              headers={'activemq.prefetchSize': config_get_int('tracer-kronos', 'prefetch_size')})
+
+
+def kronos_dataset(dataset_queue: Queue, once: bool = False, sleep_time: int = 60) -> None:
+    return_values = {'heartbeat_handler': HeartbeatHandler("kronos-dataset", 10)}
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable='kronos-dataset',
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once_kronos_dataset,
+            dataset_queue=dataset_queue,
+            return_values=return_values,
+        )
+    )
+
+    # once again for potential backlog
+    run_once_kronos_dataset(dataset_queue=dataset_queue,
+                            return_values=return_values,
+                            heartbeat_handler=return_values['heartbeat_handler'],
+                            sleep_time=sleep_time)
+
+
+def run_once_kronos_dataset(dataset_queue: Queue,
+                            return_values: dict,
+                            heartbeat_handler: "HeartbeatHandler | None",
+                            **kwargs) -> None:
+    if heartbeat_handler is None:
+        if "heartbeat_handler" not in return_values.keys():
+            return_values["heartbeat_handler"] = HeartbeatHandler("kronos-dataset", 10)
+        heartbeat_handler = return_values["heartbeat_handler"]
+
+    _, _, logger = heartbeat_handler.live()
+    len_ds = dataset_queue.qsize()
+    datasets = {}
+    dslocks = {}
+    now = time()
+    for _ in range(0, len_ds):
+        dataset = dataset_queue.get()
+        did = '%s:%s' % (dataset['scope'].internal, dataset['name'])
+        rse = dataset['rse_id']
+        if did not in datasets:
+            datasets[did] = dataset['accessed_at']
+        else:
+            datasets[did] = max(datasets[did], dataset['accessed_at'])
+
+        if rse is None:
+            continue
+        if did not in dslocks:
+            dslocks[did] = {}
+        if rse not in dslocks[did]:
+            dslocks[did][rse] = dataset['accessed_at']
+        else:
+            dslocks[did][rse] = max(dataset['accessed_at'], dslocks[did][rse])
+    logger(logging.INFO, 'fetched %d datasets from queue (%ds)' % (len_ds, time() - now))
+
+    total, failed, start = 0, 0, time()
+    for did, accessed_at in datasets.items():
+        scope, name = did.split(':')
+        scope = InternalScope(scope, from_external=False)
+        update_did = {'scope': scope, 'name': name, 'type': DIDType.DATASET, 'accessed_at': accessed_at}
+        # if update fails, put back in queue and retry next time
+        if not touch_dids((update_did,)):
+            update_did['rse_id'] = None
+            dataset_queue.put(update_did)
+            failed += 1
+        total += 1
+    logger(logging.INFO, 'update done for %d datasets, %d failed (%ds)' % (total, failed, time() - start))
+
+    total, failed, start = 0, 0, time()
+    for did, rses in dslocks.items():
+        scope, name = did.split(':')
+        scope = InternalScope(scope, from_external=False)
+        for rse, accessed_at in rses.items():
+            update_dslock = {'scope': scope, 'name': name, 'rse_id': rse, 'accessed_at': accessed_at}
+            # if update fails, put back in queue and retry next time
+            if not touch_dataset_locks((update_dslock,)):
+                dataset_queue.put(update_dslock)
+                failed += 1
+            total += 1
+    logger(logging.INFO, 'update done for %d locks, %d failed (%ds)' % (total, failed, time() - start))
+
+    total, failed, start = 0, 0, time()
+    for did, rses in dslocks.items():
+        scope, name = did.split(':')
+        scope = InternalScope(scope, from_external=False)
+        for rse, accessed_at in rses.items():
+            update_dslock = {'scope': scope, 'name': name, 'rse_id': rse, 'accessed_at': accessed_at}
+            # if update fails, put back in queue and retry next time
+            if not touch_collection_replicas((update_dslock,)):
+                dataset_queue.put(update_dslock)
+                failed += 1
+            total += 1
+    logger(logging.INFO, 'update done for %d collection replicas, %d failed (%ds)' % (total, failed, time() - start))
+
+
+def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()
+
+
+def run(
+        once: bool = False,
+        threads: int = 1,
+        sleep_time_datasets: int = 60,
+        sleep_time_files: int = 60) -> None:
+    """
+    Starts up the consumer threads
+    """
+    setup_logging(process_name='tracer-kronos')
+    logger = formatted_logger(logging.log, 'Kronos %s')
+
+    if rucio.db.sqla.util.is_old_db():
+        raise DatabaseException('Database was not updated, daemon won\'t start')
+
+    dataset_queue = Queue()
+    logger(logging.INFO, 'starting tracer consumer threads')
+
+    thread_list = []
+    for _ in range(threads):
+        krf_thread = Thread(target=kronos_file, kwargs={'once': once,
+                                                        'logger': logger,
+                                                        'sleep_time': sleep_time_files,
+                                                        'dataset_queue': dataset_queue})
+        krf_thread.start()
+        krd_thread = Thread(target=kronos_dataset, kwargs={'once': once,
+                                                           'sleep_time': sleep_time_datasets,
+                                                           'dataset_queue': dataset_queue})
+        krd_thread.start()
+        thread_list.append(krf_thread)
+        thread_list.append(krd_thread)
+
+    logger(logging.INFO, 'waiting for interrupts')
+
+    while [thread.join(timeout=3.) for thread in thread_list if thread.is_alive()]:
+        pass

rucio/daemons/transmogrifier/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.