rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/automatix/automatix.py

@@ -0,0 +1,304 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import random
+import secrets
+import tempfile
+import threading
+from configparser import NoOptionError, NoSectionError
+from datetime import datetime
+from json import load
+from os import remove, rmdir
+from typing import TYPE_CHECKING, Any, Optional
+
+import rucio.db.sqla.util
+from rucio.client import Client
+from rucio.client.uploadclient import UploadClient
+from rucio.common import exception
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import setup_logging
+from rucio.common.stopwatch import Stopwatch
+from rucio.common.types import InternalScope, LoggerFunction
+from rucio.common.utils import execute, generate_uuid
+from rucio.core.monitor import MetricManager
+from rucio.core.scope import list_scopes
+from rucio.core.vo import map_vo
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from types import FrameType
+
+
+METRICS = MetricManager(module=__name__)
+graceful_stop = threading.Event()
+DAEMON_NAME = "automatix"
+
+
+def get_data_distribution(
+        inputfile: str
+) -> tuple[dict[str, int], dict[str, Any]]:
+    with open(inputfile) as data_file:
+        data = load(data_file)
+    probabilities = {}
+    probability = 0
+    for key in data:
+        probability += data[key]["probability"]
+        probabilities[key] = probability
+    for key in list(probabilities):
+        probabilities[key] = float(probabilities[key]) / probability
+    return probabilities, data
+
+
+def choose_element(
+        probabilities: "Mapping[str, int]",
+        data: "Mapping[str, Any]"
+) -> dict[str, Any]:
+    rnd = random.uniform(0, 1)  # noqa: S311
+    prob = 0
+    for key in probabilities:
+        prob = probabilities[key]
+        if prob >= rnd:
+            return data[key]
+    return data[key]
+
+
+def generate_file(
+        fname: str,
+        size: int,
+        logger: LoggerFunction = logging.log
+) -> int:
+    cmd = "/bin/dd if=/dev/urandom of=%s bs=%s count=1" % (fname, size)
+    exitcode, out, err = execute(cmd)
+    logger(logging.DEBUG, out)
+    logger(logging.DEBUG, err)
+    return exitcode
+
+
+def generate_didname(
+        metadata: "Mapping[str, str]",
+        dsn: Optional[str],
+        did_type: str
+) -> str:
+    try:
+        did_prefix = config_get("automatix", "did_prefix")
+    except (NoOptionError, NoSectionError, RuntimeError):
+        did_prefix = ""
+    try:
+        pattern = config_get("automatix", "%s_pattern" % did_type)
+        separator = config_get("automatix", "separator")
+    except (NoOptionError, NoSectionError, RuntimeError):
+        return generate_uuid()
+    fields = pattern.split(separator)
+    file_name = ""
+    for field in fields:
+        if field == 'date':
+            field_str = str(datetime.utcnow().date())
+        elif field == 'did_prefix':
+            field_str = did_prefix
+        elif field == "dsn":
+            field_str = dsn
+        elif field == "uuid":
+            field_str = generate_uuid()
+        elif field == "randint":
+            field_str = str(secrets.choice(range(0, 100001)))
+        else:
+            field_str = metadata.get(field, None)
+            if not field_str:
+                field_str = str(secrets.choice(range(0, 100001)))
+        file_name = "%s%s%s" % (file_name, separator, field_str)
+    len_separator = len(separator)
+    return file_name[len_separator:]
+
+
+def automatix(inputfile: str, sleep_time: int, once: bool = False) -> None:
+    """
+    Creates an automatix Worker that uploads datasets to a list of rses.
+
+    :param inputfile: The input file where the parameters of the distribution is set
+    :param sleep_time: Thread sleep time after each chunk of work.
+    :param once: Run only once.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            inputfile=inputfile,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) -> bool:
+
+    _, _, logger = heartbeat_handler.live()
+    try:
+        rses = config_get_list("automatix", "rses")
+    except (NoOptionError, NoSectionError, RuntimeError):
+        logging.log(
+            logging.ERROR,
+            "Option rses not found in automatix section. Trying the legacy sites option",
+        )
+        try:
+            rses = config_get_list("automatix", "sites")
+            logging.log(
+                logging.WARNING,
+                "Option sites found in automatix section. This option will be deprecated soon. Please update your config to use rses.",
+            )
+        except (NoOptionError, NoSectionError, RuntimeError):
+            logger(logging.ERROR, "Could not load sites from configuration")
+            return True
+
+    set_metadata = config_get_bool(
+        "automatix", "set_metadata", raise_exception=False, default=True
+    )
+    dataset_lifetime = config_get_int(
+        "automatix", "dataset_lifetime", raise_exception=False, default=0
+    )
+    account = config_get("automatix", "account", raise_exception=False, default="root")
+    scope = config_get("automatix", "scope", raise_exception=False, default="test")
+    client = Client(account=account)
+    vo = map_vo(client.vo)  # type: ignore
+    filters = {"scope": InternalScope("*", vo=vo)}
+    scopes = list_scopes(filter_=filters)
+    if InternalScope(scope, vo=vo) not in scopes:
+        logger(logging.ERROR, "Scope %s does not exist. Exiting", scope)
+        return True
+
+    logger(logging.INFO, "Getting data distribution")
+    probabilities, data = get_data_distribution(inputfile)
+    logger(logging.DEBUG, "Probabilities %s", probabilities)
+
+    cycle_stopwatch = Stopwatch()
+    successes = []
+    failures = []
+    for rse in rses:
+        stopwatch = Stopwatch()
+        _, _, logger = heartbeat_handler.live()
+        tmpdir = tempfile.mkdtemp()
+        logger(logging.INFO, "Running on RSE %s", rse)
+        dic = choose_element(probabilities, data)
+        metadata = dic["metadata"]
+        try:
+            nbfiles = dic["nbfiles"]
+        except KeyError:
+            nbfiles = 2
+            logger(
+                logging.WARNING, "No nbfiles defined in the configuration, will use 2"
+            )
+        try:
+            filesize = dic["filesize"]
+        except KeyError:
+            filesize = 1000000
+            logger(
+                logging.WARNING,
+                "No filesize defined in the configuration, will use 1M files",
+            )
+        dsn = generate_didname(metadata, None, "dataset")
+        fnames = []
+        lfns = []
+        physical_fnames = []
+        files = []
+        for _ in range(nbfiles):
+            fname = generate_didname(metadata=metadata, dsn=dsn, did_type="file")
+            lfns.append(fname)
+            logger(logging.INFO, "Generating file %s in dataset %s", fname, dsn)
+            physical_fname = "%s/%s" % (tmpdir, "".join(fname.split("/")))
+            physical_fnames.append(physical_fname)
+            generate_file(physical_fname, filesize, logger=logger)
+            fnames.append(fname)
+            file_ = {
+                "did_scope": scope,
+                "did_name": fname,
+                "dataset_scope": scope,
+                "dataset_name": dsn,
+                "rse": rse,
+                "path": physical_fname,
+            }
+            if set_metadata:
+                file_["dataset_meta"] = metadata
+                if dataset_lifetime:
+                    file_["dataset_meta"]["lifetime"] = dataset_lifetime
+            files.append(file_)
+        logger(logging.INFO, "Upload %s:%s to %s", scope, dsn, rse)
+        try:
+            upload_client = UploadClient(client)
+            ret = upload_client.upload(files)
+            if ret == 0:
+                logger(logging.INFO, "%s successfully registered on %s", dsn, rse)
+                METRICS.counter(name="addnewdataset.done").inc()
+                METRICS.counter(name="addnewfile.done").inc(nbfiles)
+                METRICS.timer(name='datasetinjection').observe(stopwatch.elapsed)
+                successes.append(rse)
+            else:
+                logger(logging.INFO, "Error uploading files")
+                failures.append(rse)
+        except Exception as error:
+            logger(logging.ERROR, "Error uploading files on %s: %s", rse, str(error))
+            failures.append(rse)
+        finally:
+            for physical_fname in physical_fnames:
+                remove(physical_fname)
+            rmdir(tmpdir)
+    logger(
+        logging.INFO,
+        "It took %f seconds to upload datasets on %s RSEs: %s",
+        cycle_stopwatch.elapsed,
+        len(successes),
+        str(successes),
+    )
+    if failures:
+        logger(
+            logging.WARNING,
+            "Datasets could not be uploaded on %s RSEs: %s", len(failures), str(failures),
+        )
+    return True
+
+
+def run(
+    total_workers: int = 1,
+    once: bool = False,
+    inputfile: str = "/opt/rucio/etc/automatix.json",
+    sleep_time: int = 60,
+) -> None:
+    """
+    Starts up the automatix threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException(
+            "Database was not updated, daemon won't start"
+        )
+
+    threads = list()
+    for _ in range(total_workers):
+        kwargs = {"once": once, "sleep_time": sleep_time, "inputfile": inputfile}
+        threads.append(threading.Thread(target=automatix, kwargs=kwargs))
+    [thread.start() for thread in threads]
+    while threads[0].is_alive():
+        logging.log(logging.DEBUG, "Still %i active threads", len(threads))
+        [thread.join(timeout=3.14) for thread in threads]
+
+
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()

rucio/daemons/badreplicas/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/badreplicas/minos.py

@@ -0,0 +1,322 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+import logging
+import math
+import re
+import threading
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Optional
+
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common.config import config_get_int
+from rucio.common.exception import DatabaseException, DataIdentifierNotFound, ReplicaNotFound, UnsupportedOperation
+from rucio.common.logging import setup_logging
+from rucio.common.utils import chunks
+from rucio.core.did import get_metadata
+from rucio.core.replica import bulk_add_bad_replicas, bulk_delete_bad_pfns, declare_bad_file_replicas, get_bad_pfns, get_did_from_pfns, get_pfn_to_rse, get_replicas_state, update_replicas_states
+from rucio.core.rse import get_rse_name
+from rucio.daemons.common import run_daemon
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, BadFilesStatus, BadPFNStatus, ReplicaState
+from rucio.db.sqla.session import get_session
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+    from types import FrameType
+
+    from rucio.common.types import InternalAccount, LoggerFunction
+    from rucio.daemons.common import HeartbeatHandler
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'minos'
+
+
+def __classify_bad_pfns(
+        pfns: "Iterable[Mapping[str, Any]]"
+) -> tuple[
+    dict[tuple["InternalAccount", str, str], list[str]],
+    dict[tuple["InternalAccount", str, datetime], list[str]]
+]:
+    """
+    Function that takes a list of PFNs and classify them in 2 dictionaries : bad_replicas and temporary_unvailables
+    :param pfns: List of PFNs
+
+    :returns: Tuple (bad_replicas, temporary_unvailables)
+    """
+    states_mapping = {BadPFNStatus.BAD: BadFilesStatus.BAD,
+                      BadPFNStatus.SUSPICIOUS: BadFilesStatus.SUSPICIOUS,
+                      BadPFNStatus.TEMPORARY_UNAVAILABLE: BadFilesStatus.TEMPORARY_UNAVAILABLE}
+    bad_replicas, temporary_unvailables = {}, {}
+    for pfn in pfns:
+        path = pfn['pfn']
+        account = pfn['account']
+        reason = pfn['reason']
+        expires_at = pfn['expires_at']
+        state = states_mapping[pfn['state']]
+        if state in [BadFilesStatus.BAD, BadFilesStatus.SUSPICIOUS]:
+            if (account, reason, state) not in bad_replicas:
+                bad_replicas[(account, reason, state)] = []
+            bad_replicas[(account, reason, state)].append(path)
+        elif state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
+            if (account, reason, expires_at) not in temporary_unvailables:
+                temporary_unvailables[(account, reason, expires_at)] = []
+            temporary_unvailables[(account, reason, expires_at)].append(path)
+    return bad_replicas, temporary_unvailables
+
+
+def __clean_unknown_replicas(
+        pfns: "Iterable[str]",
+        vo: str,
+        logger: "LoggerFunction"
+) -> dict[str, dict[str, list[str]]]:
+    """
+    Identify from the list of PFNs the one that are unknown and remove them from the bad_pfns table
+    :param pfns: List of PFNs
+    :param vo: The VO name
+    :param logger: The logger
+
+    :returns: Dictionary cleaned from unkwnon replicas
+    """
+    schemes = {}
+    dict_rse = {}
+    unknown_replicas = []
+    # Splitting the PFNs by schemes
+    for pfn in pfns:
+        scheme = pfn.split(':')[0]
+        if scheme not in schemes:
+            schemes[scheme] = []
+        schemes[scheme].append(pfn)
+    for scheme in schemes:
+        _, tmp_dict_rse, tmp_unknown_replicas = get_pfn_to_rse(schemes[scheme], vo=vo)
+        for rse_id in tmp_dict_rse:
+            if rse_id not in dict_rse:
+                dict_rse[rse_id] = {}
+            if scheme not in dict_rse[rse_id]:
+                dict_rse[rse_id][scheme] = []
+            dict_rse[rse_id][scheme].extend(tmp_dict_rse[rse_id])
+        unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
+    # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
+    if unknown_replicas:
+        logger(logging.INFO, 'The following replicas are unknown and will be removed : %s', str(unknown_replicas))
+        bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
+    return dict_rse
+
+
+def __update_temporary_unavailable(
+        chunk: "Iterable[dict[str, Any]]",
+        reason: str,
+        expires_at: datetime,
+        account: "InternalAccount",
+        logger: "LoggerFunction"
+) -> None:
+    """
+    Update temporary unavailable replicas one by one
+    :param chunk: List of unavailable replicas to update
+    :param reason: Reason of the temporary unavailable replica
+    :param expires_at: Expiration date of the temporary unavailability
+    :param account: Account who declared the replica
+    :param logger: The logger
+
+    """
+    for rep in chunk:
+        logger(logging.DEBUG, 'Working on %s', str(rep))
+        try:
+            get_metadata(rep['scope'], rep['name'])
+            unavailable_states = []
+            rep_state = get_replicas_state(rep['scope'], rep['name'])
+            unavailable_states.extend(rep_state.get(ReplicaState.TEMPORARY_UNAVAILABLE, []))
+            unavailable_states.extend(rep_state.get(ReplicaState.BEING_DELETED, []))
+            unavailable_states.extend(rep_state.get(ReplicaState.BAD, []))
+            # If the replica is already not available, it is removed from the bad PFNs table
+            if rep['rse_id'] in unavailable_states:
+                logger(logging.INFO, '%s is in unavailable state. Will be removed from the list of bad PFNs', str(rep['pfn']))
+                bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+            # If the expiration date of the TEMPORARY_UNAVAILABLE is in the past, it is removed from the bad PFNs table
+            elif expires_at < datetime.utcnow():
+                logger(logging.INFO, 'PFN %s expiration time (%s) is older than now and is not in unavailable state. Removing the PFNs from bad_pfns', str(rep['pfn']), expires_at)
+                bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+            # Else update everything in the same transaction
+            else:
+                try:
+                    session = get_session()
+                    update_replicas_states([rep], nowait=False, session=session)
+                    bulk_add_bad_replicas([rep], account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
+                    bulk_delete_bad_pfns(pfns=[rep['pfn']], session=session)
+                    session.commit()  # pylint: disable=no-member
+                except Exception:
+                    logger(logging.ERROR, 'Cannot update state of %s', str(rep['pfn']))
+        except (DataIdentifierNotFound, ReplicaNotFound):
+            logger(logging.ERROR, 'Will remove %s from the list of bad PFNs', str(rep['pfn']))
+            bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+
+
+def minos(bulk: int = 1000, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Creates a Minos Worker that gets a list of bad PFNs,
+    extract the scope, name and rse_id and fill the bad_replicas table.
+
+    :param bulk: The number of requests to process.
+    :param once: Run only once.
+    :param sleep_time: Time between two cycles.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.INFO, 'Minos started')
+
+    chunk_size = config_get_int('minos', 'commit_size', default=10, raise_exception=False)  # The chunk size used for the commits
+
+    pfns = get_bad_pfns(thread=worker_number, total_threads=total_workers, limit=bulk)
+
+    # Class the PFNs into bad_replicas and temporary_unavailable
+    bad_replicas, temporary_unvailables = __classify_bad_pfns(pfns)
+
+    # Process the bad and suspicious files
+    # The scope, name, rse_id are extracted and filled into the bad_replicas table
+    for account, reason, state in bad_replicas:
+        vo = account.vo
+        pfns = bad_replicas[(account, reason, state)]
+        logger(logging.INFO, 'Declaring %s replicas with state %s and reason %s', len(pfns), str(state), reason)
+        session = get_session()
+        try:
+            dict_rse = __clean_unknown_replicas(pfns, vo, logger)
+            for rse_id, pfns_by_scheme in dict_rse.items():
+                rse = get_rse_name(rse_id=rse_id, session=None)
+                rse_vo_str = rse if vo == 'def' else '{} on VO {}'.format(rse, vo)
+                for scheme, pfns in pfns_by_scheme.items():
+                    logger(logging.DEBUG, 'Running on RSE %s with %s replicas', rse_vo_str, len(pfns))
+                    tot_chunk = int(math.ceil(len(pfns) / chunk_size))
+                    for nchunk, chunk in enumerate(chunks(pfns, chunk_size)):
+                        logger(logging.DEBUG, 'Running on %s chunk out of %s', nchunk + 1, tot_chunk)
+                        unknown_replicas = declare_bad_file_replicas(chunk, reason=reason, issuer=account, status=state, session=session)
+                        if unknown_replicas:
+                            logger(logging.DEBUG, 'Unknown replicas : %s', str(unknown_replicas))
+                        bulk_delete_bad_pfns(pfns=chunk, session=session)
+                        session.commit()  # pylint: disable=no-member
+        except (DatabaseException, DatabaseError) as error:
+            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+            else:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+            session.rollback()  # pylint: disable=no-member
+        except Exception:
+            session.rollback()  # pylint: disable=no-member
+            logger(logging.CRITICAL, 'Exception', exc_info=True)
+
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    # Now get the temporary unavailable and update the replicas states
+    for account, reason, expires_at in temporary_unvailables:
+        vo = account.vo
+        pfns = temporary_unvailables[(account, reason, expires_at)]
+        logger(logging.INFO, 'Declaring %s replicas temporary available with timeout %s and reason %s', len(pfns), str(expires_at), reason)
+        logger(logging.DEBUG, 'Extracting RSEs')
+
+        dict_rse = __clean_unknown_replicas(pfns, vo, logger)
+        for rse_id in dict_rse:
+            replicas = []
+            rse = get_rse_name(rse_id=rse_id, session=None)
+            rse_vo_str = rse if vo == 'def' else '{} on VO {}'.format(rse, vo)
+            logger(logging.DEBUG, 'Running on RSE %s', rse_vo_str)
+            for rse_id, pfns_by_scheme in dict_rse.items():
+                for scheme, pfns in pfns_by_scheme.items():
+                    for rep in get_did_from_pfns(pfns=pfns, rse_id=None, vo=vo, session=None):
+                        for pfn in rep:
+                            scope = rep[pfn]['scope']
+                            name = rep[pfn]['name']
+                            replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.TEMPORARY_UNAVAILABLE, 'pfn': pfn})
+            # The following part needs to be atomic
+            # We update the replicas states to TEMPORARY_UNAVAILABLE
+            # then insert a row in the bad_replicas table. TODO Update the row if it already exists
+            # then delete the corresponding rows into the bad_pfns table
+            logger(logging.DEBUG, 'Running on %s replicas on RSE %s', len(replicas), rse_vo_str)
+            tot_chunk = int(math.ceil(len(replicas) / float(chunk_size)))
+            session = get_session()
+            for nchunk, chunk in enumerate(chunks(replicas, chunk_size)):
+                try:
+                    logger(logging.DEBUG, 'Running on %s chunk out of %s', nchunk + 1, tot_chunk)
+                    update_replicas_states(chunk, nowait=False, session=session)
+                    bulk_add_bad_replicas(chunk, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
+                    pfns = [entry['pfn'] for entry in chunk]
+                    bulk_delete_bad_pfns(pfns=pfns, session=session)
+                    session.commit()  # pylint: disable=no-member
+                except (UnsupportedOperation, ReplicaNotFound) as error:
+                    session.rollback()  # pylint: disable=no-member
+                    logger(logging.ERROR, 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s', str(error))
+                    # Update all the replicas one by one
+                    __update_temporary_unavailable(chunk=chunk, reason=reason, expires_at=expires_at, account=account, logger=logger)
+                    session = get_session()
+                except (DatabaseException, DatabaseError) as error:
+                    if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
+                        logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                    else:
+                        logger(logging.ERROR, 'Exception', exc_info=True)
+                    session.rollback()  # pylint: disable=no-member
+                    session = get_session()
+                except Exception:
+                    session.rollback()  # pylint: disable=no-member
+                    logger(logging.CRITICAL, 'Exception', exc_info=True)
+                    session = get_session()
+    must_sleep = True
+    if len(pfns) == bulk:
+        logger(logging.INFO, 'Processed maximum number of pfns according to the bulk size. Restart immediately next cycle')
+        must_sleep = False
+        return must_sleep
+    return must_sleep
+
+
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
+    """
+    Starts up the minos threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('Will run only one iteration in a single threaded mode')
+        minos(bulk=bulk, once=once)
+    else:
+        logging.info('starting transmogrifier threads')
+        thread_list = [threading.Thread(target=minos, kwargs={'once': once,
+                                                              'sleep_time': sleep_time,
+                                                              'bulk': bulk}) for _ in range(0, threads)]
+        [thread.start() for thread in thread_list]
+        logging.info('waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while thread_list:
+            thread_list = [thread.join(timeout=3.14) for thread in thread_list if thread and thread.is_alive()]
+
+
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()