rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,15 +25,16 @@ import logging
 import re
 import threading
 import time
+from configparser import NoOptionError, NoSectionError
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Optional
 
 import rucio.db.sqla.util
-from rucio.common.config import config_get_bool
+from rucio.common.config import config_get, config_get_bool
 from rucio.common.constants import SuspiciousAvailability
-from rucio.common.exception import DatabaseException, VONotFound
+from rucio.common.exception import DatabaseException, DuplicateRule, VONotFound
 from rucio.common.logging import setup_logging
-from rucio.common.types import InternalAccount
+from rucio.common.types import InternalAccount, LoggerFunction
 from rucio.core.did import get_metadata
 from rucio.core.replica import (
     add_bad_pfns,
@@ -50,18 +50,91 @@ from rucio.daemons.common import run_daemon
 from rucio.db.sqla.util import get_db_time
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
     from types import FrameType
 
 GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'suspicious-replica-recoverer'
 
 
-def declare_suspicious_replicas_bad(once: bool = False, younger_than: int = 3, nattempts: int = 10, vos: Optional[list[str]] = None, limit_suspicious_files_on_rse: int = 5, json_file_name: str = "/opt/rucio/etc/suspicious_replica_recoverer.json", sleep_time: int = 3600, active_mode: bool = False) -> None:
+def check_suspicious_policy(policy: dict[str, str], file_metadata_datatype: str, file_metadata_scope: str) -> str:
+    match_scope = False
+    match_datatype = False
+    action = ""
+
+    if not policy.get("scope", []):
+        match_scope = True
+    for scope in policy.get("scope", []):
+        if re.match(scope, file_metadata_scope):
+            match_scope = True
+            break
+
+    if not policy.get("datatype", []):
+        match_datatype = True
+    for datatype in policy.get("datatype", []):
+        if re.match(datatype, file_metadata_datatype):
+            match_datatype = True
+            break
+
+    if match_scope and match_datatype:
+        action = policy["action"]
+
+    return action
+
+
+def parse_replica_datatype(
+        did_name_expression: Optional[str],
+        file_name: str,
+        rse_key: str,
+        logger: LoggerFunction) -> Optional[str]:
+
+    file_metadata_datatype = None
+    if did_name_expression is not None:
+        logger(logging.WARNING, f'Using {did_name_expression} as regex to determine file datatype for file: {file_name} on rse: {rse_key}')
+        try:
+            file_metadata_datatype_reg = re.compile(did_name_expression).search(file_name)
+            if file_metadata_datatype_reg is not None:
+                file_metadata_datatype = file_metadata_datatype_reg[0]
+        except (re.error, TypeError) as e:
+            logger(logging.WARNING, f'Could not determine datatype using regex {did_name_expression} on file: {file_name}, ignoring: {e}')
+
+    return file_metadata_datatype
+
+
+def _read_from_config(logger: LoggerFunction) -> tuple[str, bool, Optional[str]]:
+    rule_suspicious_rse_expression = config_get("replicarecoverer", "rule_rse_expression", raise_exception=False, default='type=SCRATCHDISK')
+    logger(logging.INFO, f"Recovering with the rse expression: {rule_suspicious_rse_expression}")
+
+    use_file_metadata = config_get_bool("replicarecoverer", "use_file_metadata", raise_exception=False, default=True)
+    did_name_expression = None
+
+    if not use_file_metadata:
+        try:
+            did_name_expression = config_get('replicarecoverer', 'did_name_expression', raise_exception=True)
+            logger(logging.INFO, f"Setting replica datatype from did name with regex expression: {did_name_expression}")
+
+        except (NoOptionError, NoSectionError) as e:
+            logger(logging.WARNING, f"Cannot use option 'use_file_metadata' without setting a replacement expression with 'did_name_expression'- {e}")
+            use_file_metadata = True
+
+    logger(logging.INFO, f"Recovering with the rse expression: {rule_suspicious_rse_expression}")
+    return rule_suspicious_rse_expression, use_file_metadata, did_name_expression
+
+
+def declare_suspicious_replicas_bad(
+        once: bool = False,
+        younger_than: int = 5,
+        nattempts: int = 5,
+        vos: "Optional[Sequence[str]]" = None,
+        limit_suspicious_files_on_rse: int = 5,
+        json_file_name: str = "/opt/rucio/etc/suspicious_replica_recoverer.json",
+        sleep_time: int = 3600,
+        active_mode: bool = False) -> None:
     """
     Main loop to check for available replicas which are labeled as suspicious.
 
     Gets a list of suspicious replicas that are listed as AVAILABLE in 'replicas' table
-    and available on other RSE. Finds surls of these replicas and declares them as bad.
+    and available on other RSE. Finds pfns of these replicas and declares them as bad.
     Replicas that are the last remaining copy of a file have additional checks (checksum
     comparison, etc.) before being declared bad.
 
@@ -99,10 +172,10 @@ def declare_suspicious_replicas_bad(once: bool = False, younger_than: int = 3, n
     )
 
 
-def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Optional[list[str]], limit_suspicious_files_on_rse: int, json_file_name: str, active_mode: int, **_kwargs) -> bool:
-
-    worker_number, total_workers, logger = heartbeat_handler.live()
+def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: "Optional[Sequence[str]]", limit_suspicious_files_on_rse: int, json_file_name: str, active_mode: int, **_kwargs) -> bool:
 
+    _, _, logger = heartbeat_handler.live()
+    rule_suspicious_rse_expression, use_file_metadata, did_name_expression = _read_from_config(logger=logger)
     vos = vos or []
 
     multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
@@ -126,7 +199,7 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
         json_file = open(json_file_name, mode="r")
         logger(logging.INFO, "JSON file has been opened.")
     except:
-        logger(logging.WARNING, "An error occured while trying to open the JSON file.")
+        logger(logging.WARNING, "An error occurred while trying to open the JSON file.")
         must_sleep = True
         return must_sleep
 
@@ -175,8 +248,8 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
         for rse in rse_list:
             time_start_rse = time.time()
             rse_expr = rse['rse']
-            cnt_surl_not_found = 0
-            cnt_surl_not_found_nattempts_1 = 0
+            cnt_pfn_not_found = 0
+            cnt_pfn_not_found_nattempts_1 = 0
             if rse_expr not in recoverable_replicas[vo]:
                 recoverable_replicas[vo][rse_expr] = {}
             if rse_expr not in replicas_nattempts_1[vo]:
@@ -210,16 +283,16 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
                         scope = replica['scope']
                         rep_name = replica['name']
                         rse_id = replica['rse_id']
-                        surl_not_found = True
+                        pfn_not_found = True
                         for rep in list_replicas([{'scope': scope, 'name': rep_name}]):
                             for rse_ in rep['rses']:
                                 if rse_ == rse_id:
-                                    recoverable_replicas[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'surl': rep['rses'][rse_][0], 'available_elsewhere': True}
-                                    surl_not_found = False
+                                    recoverable_replicas[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'pfn': rep['rses'][rse_][0], 'available_elsewhere': True}
+                                    pfn_not_found = False
 
-                        if surl_not_found:
-                            cnt_surl_not_found += 1
-                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no surls were found.', rep_name, rse_expr)
+                        if pfn_not_found:
+                            cnt_pfn_not_found += 1
+                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no pfns were found.', rep_name, rse_expr)
 
             if suspicious_replicas_last_copy:
                 for replica in suspicious_replicas_last_copy:
@@ -227,14 +300,14 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
                         scope = replica['scope']
                         rep_name = replica['name']
                         rse_id = replica['rse_id']
-                        surl_not_found = True
+                        pfn_not_found = True
                         # Should only return one rse, as there is only one replica remaining
                         for rep in list_replicas([{'scope': scope, 'name': rep_name}]):
-                            recoverable_replicas[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'surl': rep['rses'][rse_id][0], 'available_elsewhere': False}
-                            surl_not_found = False
-                        if surl_not_found:
-                            cnt_surl_not_found += 1
-                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no surls were found.', rep_name, rse_expr)
+                            recoverable_replicas[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'pfn': rep['rses'][rse_id][0], 'available_elsewhere': False}
+                            pfn_not_found = False
+                        if pfn_not_found:
+                            cnt_pfn_not_found += 1
+                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no pfns were found.', rep_name, rse_expr)
 
             if suspicious_replicas_nattempts_1:
                 for replica in suspicious_replicas_nattempts_1:
@@ -242,27 +315,27 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
                         scope = replica['scope']
                         rep_name = replica['name']
                         rse_id = replica['rse_id']
-                        surl_not_found = True
+                        pfn_not_found = True
                         for rep in list_replicas([{'scope': scope, 'name': rep_name}]):
                             for rse_ in rep['rses']:
                                 if rse_ == rse_id:
-                                    replicas_nattempts_1[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'surl': rep['rses'][rse_][0], 'available_elsewhere': True}
-                                    surl_not_found = False
-                        if surl_not_found:
-                            cnt_surl_not_found_nattempts_1 += 1
-                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no surls were found.', rep_name, rse_expr)
+                                    replicas_nattempts_1[vo][rse_expr][rep_name] = {'name': rep_name, 'rse_id': rse_id, 'scope': scope, 'pfn': rep['rses'][rse_][0], 'available_elsewhere': True}
+                                    pfn_not_found = False
+                        if pfn_not_found:
+                            cnt_pfn_not_found_nattempts_1 += 1
+                            logger(logging.WARNING, 'Skipping suspicious replica %s on %s, no pfns were found.', rep_name, rse_expr)
 
             logger(logging.INFO, 'Suspicious replica query took %s seconds on %s and found %i suspicious replica(s) with a minimum of nattempts=%i. The pfns for %s/%s replicas were found.',
                    time.time() - time_start_rse,
                    rse_expr,
                    len(suspicious_replicas_avail_elsewhere) + len(suspicious_replicas_last_copy),
                    nattempts,
-                   len(suspicious_replicas_avail_elsewhere) + len(suspicious_replicas_last_copy) - cnt_surl_not_found,
+                   len(suspicious_replicas_avail_elsewhere) + len(suspicious_replicas_last_copy) - cnt_pfn_not_found,
                    len(suspicious_replicas_avail_elsewhere) + len(suspicious_replicas_last_copy))
 
             logger(logging.INFO, 'A total of %i replicas with exactly nattempts=1 were found. The pfns for %s/%s replicas were found.',
                    len(suspicious_replicas_nattempts_1),
-                   len(suspicious_replicas_nattempts_1) - cnt_surl_not_found_nattempts_1,
+                   len(suspicious_replicas_nattempts_1) - cnt_pfn_not_found_nattempts_1,
                    len(suspicious_replicas_nattempts_1))
 
             if len(suspicious_replicas_avail_elsewhere) + len(suspicious_replicas_last_copy) != 0:
@@ -278,24 +351,80 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
         logger(logging.INFO, 'All RSEs have been checked for suspicious replicas. Total time: %s seconds.', time.time() - start)
 
         # Checking that everything is still working properly
-        worker_number, total_workers, logger = heartbeat_handler.live()
+        _, _, logger = heartbeat_handler.live()
 
         logger(logging.INFO, 'Create rules for replicas with nattempts=1.')
 
-        for rse_key in replicas_nattempts_1[vo]:
+        # Create as many rules as necessary for the replicas to be picked up by the daemon on the next run
+        # Create rules only for replicas that can be declared bad.
+        # Replicas from the auditor should be declared bad regardless of suspicious declarations, so no rules necessary.
+        for rse_key in list(replicas_nattempts_1[vo].keys()):
+            if not replicas_nattempts_1[vo][rse_key]:
+                # This is needed for testing purposes.
+                continue
+            files_to_be_declared_bad_nattempts_1 = []
             dids_nattempts_1 = []
-            for replica_values in replicas_nattempts_1[vo][rse_key].values():
-                dids = {'scope': replica_values['scope'], 'name': replica_values['name'], 'rse': rse_key}
-                dids_nattempts_1.append(dids)
-
+            # Get the rse_id by going to one of the suspicious replicas from that RSE and reading it from there
+            rse_id = list(replicas_nattempts_1[vo][rse_key].values())[0]['rse_id']
+            for replica_key in replicas_nattempts_1[vo][rse_key].keys():
+                from_auditor = False
+                file_scope = replicas_nattempts_1[vo][rse_key][replica_key]["scope"]
+                file_name = replicas_nattempts_1[vo][rse_key][replica_key]["name"]
+                file_metadata = get_metadata(file_scope, file_name)
+                replicas_nattempts_1[vo][rse_key][replica_key]["datatype"] = str(file_metadata["datatype"])
+
+                # Auditor
+                suspicious_reason = get_suspicious_reason(replicas_nattempts_1[vo][rse_key][replica_key]["rse_id"], file_scope, file_name, nattempts)
+                for reason in suspicious_reason:
+                    if "auditor" in reason["reason"].lower():
+                        from_auditor = True
+                        files_to_be_declared_bad_nattempts_1.append(replicas_nattempts_1[vo][rse_key][replica_key])
+                        break
+
+                # Bad
+                if not from_auditor:
+                    if (file_name.startswith("log.")) or (file_name.startswith("user")):
+                        # Don't keep log files or user files
+                        files_to_be_declared_bad_nattempts_1.append(replicas_nattempts_1[vo][rse_key][replica_key])
+                        action = ""
+                    else:
+                        # Deal with replicas based on their metadata.
+                        if (file_metadata["datatype"] is None) and (use_file_metadata):  # "None" type has no function "split()"
+                            logger(logging.WARNING, "RSE: %s, replica name %s, pfn %s: Replica does not have a data type associated with it. No action will be taken.",
+                                   rse_key, replica_key, replicas_nattempts_1[vo][rse_key][replica_key]['pfn'])
+                            continue
+                        file_metadata_datatype = str(file_metadata["datatype"])
+                        if not use_file_metadata:
+                            file_metadata_datatype = parse_replica_datatype(did_name_expression, file_name, rse_key, logger)
+                        file_metadata_scope = str(file_metadata["scope"])
+                        action = ""
+                        if file_metadata_datatype:
+                            # Some files don't have a datatype. They should be ignored.
+                            for policy in json_data:
+                                action = check_suspicious_policy(policy=policy, file_metadata_datatype=file_metadata_datatype, file_metadata_scope=file_metadata_scope)
+                                if action:
+                                    logger(logging.INFO, "The action that will be performed is %s", action)
+                                    break
+                    if action:
+                        # Rules will be created for these replicas.
+                        dids = {'scope': file_scope, 'name': file_name, 'rse': rse_key}
+                        dids_nattempts_1.append(dids)
             if active_mode:
-                # Create as many rules as necessary for the replicas to be picked up by the daemon on the next run
                 if len(dids_nattempts_1) > 0:
-                    add_rule(dids=dids_nattempts_1, account=InternalAccount('root', vo=vo), copies=nattempts, rse_expression='type=SCRATCHDISK', grouping=None, weight=None, lifetime=24 * 3600, locked=False, subscription_id=None)
-
-                    logger(logging.INFO, 'Rules have been created for %i replicas on %s.', len(dids_nattempts_1), rse_key)
+                    try:
+                        add_rule(dids=dids_nattempts_1, account=InternalAccount('root', vo=vo), copies=nattempts, rse_expression=rule_suspicious_rse_expression, grouping=None, weight=None, lifetime=5 * 24 * 3600, locked=False, subscription_id=None)
+                        logger(logging.INFO, 'Rules have been created for %i replicas on %s.', len(dids_nattempts_1), rse_key)
+                    except DuplicateRule:
+                        logger(logging.INFO, 'Tried to create rules on %s, but it already exists.', rse_key)
                 else:
-                    logger(logging.INFO, 'No replicas on %s with nattempts=1.', rse_key)
+                    logger(logging.INFO, 'No replicas on %s have nattmepts=1, so no rules have been created.', rse_key)
+                if len(files_to_be_declared_bad_nattempts_1) > 0:
+                    logger(logging.INFO, 'Ready to declare %s bad replica(s) with nattempts=1 on %s (RSE id: %s).', len(files_to_be_declared_bad_nattempts_1), rse_key, str(rse_id))
+                    declare_bad_file_replicas(replicas=files_to_be_declared_bad_nattempts_1, reason='Suspicious. Automatic recovery.', issuer=InternalAccount('root', vo=vo), session=None)
+                else:
+                    logger(logging.INFO, 'No suspicious replica(s) with nattempts=1 on %s (RSE id: %s) have been declared bad.', rse_key, str(rse_id))
+            else:
+                logger(logging.INFO, 'No replicas on %s with nattempts=1.', rse_key)
 
         logger(logging.INFO, 'Begin check for problematic RSEs.')
         time_start_check_probl = time.time()
@@ -306,17 +435,17 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
         for rse_key in list(recoverable_replicas[vo].keys()):
             if len(recoverable_replicas[vo][rse_key].values()) > limit_suspicious_files_on_rse:
                 list_problematic_rses.append(rse_key)
-                surls_list = []
+                pfns_list = []
                 for replica_value in recoverable_replicas[vo][rse_key].values():
-                    surls_list.append(replica_value['surl'])
+                    pfns_list.append(replica_value['pfn'])
 
                 if active_mode:
-                    add_bad_pfns(pfns=surls_list, account=InternalAccount('root', vo=vo), state='TEMPORARY_UNAVAILABLE', expires_at=datetime.utcnow() + timedelta(days=3))
+                    add_bad_pfns(pfns=pfns_list, account=InternalAccount('root', vo=vo), state='TEMPORARY_UNAVAILABLE', expires_at=datetime.utcnow() + timedelta(days=3))
 
                 logger(logging.INFO, "%s is problematic (more than %s suspicious replicas). Send a Jira ticket for the RSE (to be implemented).", rse_key, limit_suspicious_files_on_rse)
                 logger(logging.INFO, "The following files on %s have been marked as TEMPORARILY UNAVAILABLE:", rse_key)
                 for replica_values in recoverable_replicas[vo][rse_key].values():
-                    logger(logging.INFO, 'Temporarily unavailable: RSE: %s    Scope: %s    Name: %s    PFN: %s', rse_key, replica_values['scope'], replica_values['name'], replica_values['surl'])
+                    logger(logging.INFO, 'Temporarily unavailable: RSE: %s    Scope: %s    Name: %s    PFN: %s', rse_key, replica_values['scope'], replica_values['name'], replica_values['pfn'])
                 # Remove the RSE from the dictionary as it has been dealt with.
                 del recoverable_replicas[vo][rse_key]
 
@@ -325,16 +454,17 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
             logger(logging.INFO, "%s", rse)
 
         # Checking that everything is still working properly
-        worker_number, total_workers, logger = heartbeat_handler.live()
+        _, _, logger = heartbeat_handler.live()
 
         auditor = 0
         checksum = 0
 
-        # Label suspicious replicas as bad if they have oher copies on other RSEs (that aren't also marked as suspicious).
+        # Label suspicious replicas as bad if they have other copies on other RSEs (that aren't also marked as suspicious).
         # If they are the last remaining copies, deal with them differently.
         for rse_key in list(recoverable_replicas[vo].keys()):
             files_to_be_declared_bad = []
             files_to_be_ignored = []
+            files_dry_run_monitoring = []
             # Remove RSEs from dictionary that don't have any suspicious replicas
             if len(recoverable_replicas[vo][rse_key]) == 0:
                 del recoverable_replicas[vo][rse_key]
@@ -342,86 +472,87 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
             # Get the rse_id by going to one of the suspicious replicas from that RSE and reading it from there
             rse_id = list(recoverable_replicas[vo][rse_key].values())[0]['rse_id']
             for replica_key in list(recoverable_replicas[vo][rse_key].keys()):
+                from_auditor = False
                 file_scope = recoverable_replicas[vo][rse_key][replica_key]["scope"]
                 file_name = recoverable_replicas[vo][rse_key][replica_key]["name"]
                 file_metadata = get_metadata(file_scope, file_name)
                 recoverable_replicas[vo][rse_key][replica_key]["datatype"] = str(file_metadata["datatype"])
-                if recoverable_replicas[vo][rse_key][replica_key]['available_elsewhere'] is True:
-                    # Replicas with other copies on at least one other RSE can safely be labeled as bad
-                    files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
-                    # Remove replica from dictionary
-                    del recoverable_replicas[vo][rse_key][replica_key]
-                elif recoverable_replicas[vo][rse_key][replica_key]['available_elsewhere'] is False:
-                    if (file_name.startswith("log.")) or (file_name.startswith("user")):
-                        # Don't keep log files or user files
-                        files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
-                        del recoverable_replicas[vo][rse_key][replica_key]
-                    else:
-                        # Deal with replicas based on their metadata.
-                        if file_metadata["datatype"] is None:  # "None" type has no function "split()"
-                            files_to_be_ignored.append(recoverable_replicas[vo][rse_key][replica_key])
-                            logger(logging.WARNING, "RSE: %s, replica name %s, surl %s: Replica does not have a data type associated with it. No action will be taken.",
-                                   rse_key, replica_key, recoverable_replicas[vo][rse_key][replica_key]['surl'])
-                            continue
 
-                        file_metadata_datatype = str(file_metadata["datatype"])
-                        file_metadata_scope = str(file_metadata["scope"])
-                        action = ""
-                        if file_metadata_datatype:
-                            # Some files don't have a datatype. They should be ignored.
-                            for policy in json_data:
-                                match_scope = False
-                                match_datatype = False
-
-                                if not policy.get("scope", []):
-                                    match_scope = True
-                                for scope in policy.get("scope", []):
-                                    if re.match(scope, file_metadata_scope):
-                                        match_scope = True
-                                        break
+                suspicious_reason = get_suspicious_reason(recoverable_replicas[vo][rse_key][replica_key]["rse_id"],
+                                                          file_scope,
+                                                          file_name,
+                                                          nattempts)
+                for reason in suspicious_reason:
+                    if "auditor" in reason["reason"].lower():
+                        auditor += 1
+                        files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
+                        from_auditor = True
+                        break
 
-                                if not policy.get("datatype", []):
-                                    match_datatype = True
-                                for datatype in policy.get("datatype", []):
-                                    if re.match(datatype, file_metadata_datatype):
-                                        match_datatype = True
+                if not from_auditor:
+                    if recoverable_replicas[vo][rse_key][replica_key]['available_elsewhere'] is True:
+                        # Replicas with other copies on at least one other RSE can safely be labeled as bad
+                        files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
+                        # Remove replica from dictionary
+                        del recoverable_replicas[vo][rse_key][replica_key]
+                    elif recoverable_replicas[vo][rse_key][replica_key]['available_elsewhere'] is False:
+                        if (file_name.startswith("log.")) or (file_name.startswith("user")):
+                            # Don't keep log files or user files
+                            files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
+                            del recoverable_replicas[vo][rse_key][replica_key]
+                        else:
+                            # Deal with replicas based on their metadata.
+                            if (file_metadata["datatype"] is None) and use_file_metadata:  # "None" type has no function "split()"
+                                files_to_be_ignored.append(recoverable_replicas[vo][rse_key][replica_key])
+                                logger(logging.WARNING, "RSE: %s, replica name %s, pfn %s: Replica does not have a data type associated with it. No action will be taken.",
+                                       rse_key, replica_key, recoverable_replicas[vo][rse_key][replica_key]['pfn'])
+                                continue
+
+                            file_metadata_datatype = str(file_metadata["datatype"])
+                            if not use_file_metadata:
+                                file_metadata_datatype = parse_replica_datatype(did_name_expression, file_name, rse_key, logger)
+                            file_metadata_scope = str(file_metadata["scope"])
+                            action = ""
+                            if file_metadata_datatype:
+                                # Some files don't have a datatype. They should be ignored.
+                                for policy in json_data:
+                                    action = check_suspicious_policy(policy=policy, file_metadata_datatype=file_metadata_datatype, file_metadata_scope=file_metadata_scope)
+                                    if action:
+                                        logger(logging.INFO, "The action that will be performed is %s", action)
                                         break
 
-                                if match_scope and match_datatype:
-                                    action = policy["action"]
-                                    logger(logging.INFO, "The action that will be performed is %s", action)
-                                    break
-
-                        if not action:
-                            logger(logging.WARNING, "No recognised actions (ignore/declare bad) found in policy file (etc/suspicious_replica_recoverer.json). Replica will be ignored by default.")
-
-                        if action:
-                            if action == "ignore":
+                            if not action:
+                                logger(logging.WARNING, "No recognised actions (ignore/declare bad) found in policy file (etc/suspicious_replica_recoverer.json). Replica will be ignored by default.")
+
+                            if action:
+                                if action == "dry run":
+                                    # Monitoring purposes: Will look like a file has been declared bad, even though no
+                                    # actions will be taken.
+                                    files_dry_run_monitoring.append(recoverable_replicas[vo][rse_key][replica_key])
+                                elif action == "ignore":
+                                    files_to_be_ignored.append(recoverable_replicas[vo][rse_key][replica_key])
+                                elif action == "declare bad":
+                                    suspicious_reason = get_suspicious_reason(recoverable_replicas[vo][rse_key][replica_key]["rse_id"],
+                                                                              file_scope,
+                                                                              file_name,
+                                                                              nattempts)
+                                    for reason in suspicious_reason:
+                                        if "checksum" in reason["reason"].lower():
+                                            checksum += 1
+                                            files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
+                                            break
+                            else:
+                                # If no policy has been set, default to ignoring the file (no action taken).
                                 files_to_be_ignored.append(recoverable_replicas[vo][rse_key][replica_key])
-                            elif action == "declare bad":
-                                suspicious_reason = get_suspicious_reason(recoverable_replicas[vo][rse_key][replica_key]["rse_id"],
-                                                                          file_scope,
-                                                                          file_name,
-                                                                          nattempts)
-                                for reason in suspicious_reason:
-                                    if "auditor" in reason["reason"].lower():
-                                        auditor += 1
-                                        files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
-                                        break
-                                    elif "checksum" in reason["reason"].lower():
-                                        checksum += 1
-                                        files_to_be_declared_bad.append(recoverable_replicas[vo][rse_key][replica_key])
-                                        break
-                        else:
-                            # If no policy has been set, default to ignoring the file (no action taken).
-                            files_to_be_ignored.append(recoverable_replicas[vo][rse_key][replica_key])
 
             logger(logging.INFO, '(%s) Remaining replicas (pfns) that will be ignored:', rse_key)
             for i in files_to_be_ignored:
-                logger(logging.INFO, 'Ignore: RSE: %s    Scope: %s    Name: %s    Datatype: %s    PFN: %s', rse_key, i["scope"], i["name"], i["datatype"], i["surl"])
+                logger(logging.INFO, 'Ignore: RSE: %s    Scope: %s    Name: %s    Datatype: %s    PFN: %s', rse_key, i["scope"], i["name"], i["datatype"], i["pfn"])
             logger(logging.INFO, '(%s) Remaining replica (pfns) that will be declared BAD:', rse_key)
             for i in files_to_be_declared_bad:
-                logger(logging.INFO, 'Declare bad: RSE: %s    Scope: %s    Name: %s    Datatype: %s    PFN: %s', rse_key, i["scope"], i["name"], i["datatype"], i["surl"])
+                logger(logging.INFO, 'Declare bad: RSE: %s    Scope: %s    Name: %s    Datatype: %s    PFN: %s', rse_key, i["scope"], i["name"], i["datatype"], i["pfn"])
+            for i in files_dry_run_monitoring:
+                logger(logging.INFO, 'Declare bad (dry run): RSE: %s    Scope: %s    Name: %s    Datatype: %s    PFN: %s', rse_key, i["scope"], i["name"], i["datatype"], i["pfn"])
 
             if files_to_be_declared_bad:
                 logger(logging.INFO, 'Ready to declare %s bad replica(s) on %s (RSE id: %s).', len(files_to_be_declared_bad), rse_key, str(rse_id))
@@ -446,7 +577,15 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
     return must_sleep
 
 
-def run(once: bool = False, younger_than: int = 3, nattempts: int = 10, vos: list[str] = None, limit_suspicious_files_on_rse: int = 5, json_file_name: str = "/opt/rucio/etc/suspicious_replica_recoverer.json", sleep_time: int = 3600, active_mode: bool = False) -> None:
+def run(
+        once: bool = False,
+        younger_than: int = 5,
+        nattempts: int = 5,
+        vos: "Optional[Sequence[str]]" = None,
+        limit_suspicious_files_on_rse: int = 5,
+        json_file_name: str = "/opt/rucio/etc/suspicious_replica_recoverer.json",
+        sleep_time: int = 3600,
+        active_mode: bool = False) -> None:
     """
     Starts up the Suspicious-Replica-Recoverer threads.
     """

rucio/{api → daemons/rsedecommissioner}/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/daemons/rsedecommissioner/config.py

@@ -0,0 +1,81 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Functions to manage decommissioning configurations."""
+
+from enum import Enum
+from typing import Any
+
+from rucio.common.constants import RseAttr
+from rucio.core.rse import add_rse_attribute, get_rse_attribute
+
+
+class DecommissioningStatus(Enum):
+    """Decommissioning status flags."""
+
+    PROCESSING = 'processing'
+    DONE = 'done'
+    SUSPENDED = 'suspended'
+
+
+class InvalidStatusName(Exception):
+    """Exception for invalid decommissioning status name set from command line."""
+
+
+def config_to_attr(config: dict[str, Any]) -> str:
+    """Form the attribute string from a config dictionary.
+
+    :param config: Decommissioning configuration dictionary.
+    :returns: Comma-separated key-value string encoding the configuration.
+    """
+    attr = f'profile={config["profile"].value}'
+    if config.get('move_dest'):
+        attr += f',move_dest={config["move_dest"]}'
+    attr += f',status={config["status"].value}'
+
+    return attr
+
+
+def attr_to_config(attr: str) -> dict[str, Any]:
+    """Form the config dictionary from an attribute string.
+
+    :param attr: Comma-separated key-value string encoding the configuration.
+    :returns: Decommissioning configuration dictionary.
+    """
+    # The decommission attribute is a comma-separated list of key=value settings
+    config: dict[str, Any] = dict(map(lambda s: s.split('='), attr.split(',')))
+    if 'status' in config:
+        try:
+            config['status'] = DecommissioningStatus[config['status'].upper()]
+        except KeyError as exc:
+            raise InvalidStatusName() from exc
+    else:
+        config['status'] = DecommissioningStatus.PROCESSING
+
+    return config
+
+
+def set_status(
+    rse_id: str,
+    status: DecommissioningStatus
+) -> None:
+    """Update the decommission attribute of the RSE.
+
+    :param rse_id: RSE ID.
+    :param status: RSE decommissioning status.
+    """
+    config = attr_to_config(get_rse_attribute(rse_id, RseAttr.DECOMMISSION))
+    config['status'] = status
+    # add_rse_attribute can handle updating existing entries too
+    add_rse_attribute(rse_id, RseAttr.DECOMMISSION, config_to_attr(config))