rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/abacus/rse.py

@@ -0,0 +1,117 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-RSE is a daemon to update RSE counters.
+"""
+
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.common.utils import get_thread_with_periodic_running_function
+from rucio.core.rse_counter import fill_rse_counter_history_table, get_updated_rse_counters, update_rse_counter
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-rse'
+
+
+def rse_update(
+        once: bool = False,
+        sleep_time: int = 10
+) -> None:
+    """
+    Main loop to check and update the RSE Counters.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=run_once,
+    )
+
+
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        **_kwargs
+) -> None:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    # Select a bunch of rses for to update for this worker
+    start = time.time()  # NOQA
+    rse_ids = get_updated_rse_counters(total_workers=total_workers,
+                                       worker_number=worker_number)
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(rse_ids)))
+
+    # If the list is empty, sent the worker to sleep
+    if not rse_ids:
+        logger(logging.INFO, 'did not get any work')
+        return
+
+    for rse_id in rse_ids:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_rse_counter(rse_id=rse_id)
+        logger(logging.DEBUG, 'update of rse "%s" took %f' % (rse_id, time.time() - start_time))
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(
+        once: bool = False,
+        threads: int = 1,
+        fill_history_table: bool = False,
+        sleep_time: int = 10
+) -> None:
+    """
+    Starts up the Abacus-RSE threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        rse_update(once)
+    else:
+        logging.info('main: starting threads')
+        thread_list = [threading.Thread(target=rse_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                       range(0, threads)]
+        if fill_history_table:
+            thread_list.append(get_thread_with_periodic_running_function(3600, fill_rse_counter_history_table, graceful_stop))
+        [t.start() for t in thread_list]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]

rucio/daemons/atropos/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/atropos/atropos.py

@@ -0,0 +1,242 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import functools
+import logging
+import random
+import threading
+from sys import exc_info
+from traceback import format_exception
+from typing import TYPE_CHECKING
+
+import rucio.core.lifetime_exception
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.exception import InvalidRSEExpression, RuleNotFound
+from rucio.common.logging import setup_logging
+from rucio.core.did import set_metadata
+from rucio.core.lock import get_dataset_locks
+from rucio.core.rse import get_rse_name, get_rse_vo
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.core.rule import get_rules_beyond_eol, update_rule
+from rucio.daemons.common import run_daemon
+from rucio.db.sqla.constants import LifetimeExceptionsState
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.daemons.common import HeartbeatHandler
+
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'atropos'
+
+
+def atropos(
+        date_check: datetime.datetime,
+        dry_run: bool,
+        grace_period: int,
+        purge_replicas: bool,
+        spread_period: bool,
+        unlock: bool,
+        once: bool,
+        sleep_time: int
+) -> None:
+    """
+    Creates an Atropos Worker that gets a list of rules which have an eol_at expired and delete them.
+
+    :param grace_period: The grace_period for the rules.
+    :param once: Run only once.
+    :param sleep_time: Thread sleep time after each chunk of work.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=DAEMON_NAME,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            date_check=date_check,
+            dry_run=dry_run,
+            grace_period=grace_period,
+            purge_replicas=purge_replicas,
+            spread_period=spread_period,
+            unlock=unlock
+        )
+    )
+
+
+def run_once(
+        heartbeat_handler: 'HeartbeatHandler',
+        activity: None,    # NOQA: ARG001
+        date_check: datetime.datetime,
+        dry_run: bool,
+        grace_period: int,
+        purge_replicas: bool,
+        spread_period: int,
+        unlock: bool,
+) -> None:
+    """An iteration of an Atropos worker."""
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.DEBUG, 'Starting worker')
+    if not dry_run and date_check > datetime.datetime.utcnow():
+        logger(logging.ERROR, 'Atropos cannot run in non-dry-run mode for date in the future')
+        return
+
+    # Process the list of approved exceptions. In case a DID has
+    # multiple exceptions, the one with the expiration date further in
+    # the future is what matters.
+    summary = {}
+    lifetime_exceptions = {}
+    for excep in rucio.core.lifetime_exception.list_exceptions(exception_id=None, states=[LifetimeExceptionsState.APPROVED, ], session=None):
+        key = '{}:{}'.format(excep['scope'].internal, excep['name'])
+        if key not in lifetime_exceptions:
+            lifetime_exceptions[key] = excep['expires_at']
+        elif lifetime_exceptions[key] < excep['expires_at']:
+            lifetime_exceptions[key] = excep['expires_at']
+    logger(logging.DEBUG, '%d active exceptions', len(lifetime_exceptions))
+
+    rand = random.Random(worker_number)   # noqa: S311
+
+    try:
+        rules = get_rules_beyond_eol(date_check, worker_number, total_workers, session=None)
+        logger(logging.INFO, '%d rules to process', len(rules))
+        for rule_idx, rule in enumerate(rules, start=1):
+            did = '%s:%s' % (rule.scope, rule.name)
+            did_key = '{}:{}'.format(rule.scope.internal, rule.name)
+            logger(logging.DEBUG, 'Working on rule %s on DID %s on %s', rule.id, did, rule.rse_expression)
+
+            if (rule_idx % 1000) == 0:
+                logger(logging.INFO, '%s/%s rules processed', rule_idx, len(rules))
+
+            # We compute the expected eol_at
+            try:
+                rses = parse_expression(rule.rse_expression, filter_={'vo': rule.account.vo})
+            except InvalidRSEExpression:
+                logger(logging.WARNING, 'Rule %s has an RSE expression that results in an empty set: %s', rule.id, rule.rse_expression)
+                continue
+            eol_at = rucio.core.lifetime_exception.define_eol(rule.scope, rule.name, rses)
+            if eol_at != rule.eol_at:
+                logger(logging.WARNING, 'The computed eol %s differs from the one recorded %s for rule %s on %s at %s',
+                       eol_at, rule.eol_at, rule.id, did, rule.rse_expression)
+                try:
+                    update_rule(rule.id, options={'eol_at': eol_at})
+                except RuleNotFound:
+                    logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                    continue
+
+            # Check the exceptions
+            if did_key in lifetime_exceptions:
+                if eol_at > lifetime_exceptions[did_key]:
+                    logger(logging.INFO, 'Rule %s on DID %s on %s has longer expiration date than the one requested : %s',
+                           rule.id, did, rule.rse_expression, lifetime_exceptions[did_key])
+                else:
+                    # If eol_at < requested extension, update eol_at
+                    logger(logging.INFO, 'Updating rule %s on DID %s on %s according to the exception till %s',
+                           rule.id, did, rule.rse_expression, lifetime_exceptions[did_key])
+                    eol_at = lifetime_exceptions[did_key]
+                    try:
+                        update_rule(rule.id, options={'eol_at': lifetime_exceptions[did_key]})
+                    except RuleNotFound:
+                        logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                        continue
+
+            # Now check that the new eol_at is expired
+            if eol_at and eol_at <= date_check:
+                set_metadata(scope=rule.scope, name=rule.name, key='eol_at', value=eol_at)
+                no_locks = True
+                for lock in get_dataset_locks(rule.scope, rule.name):
+                    if lock['rule_id'] == rule[4]:
+                        no_locks = False
+                        if lock['rse_id'] not in summary:
+                            summary[lock['rse_id']] = {}
+                        if did_key not in summary[lock['rse_id']]:
+                            summary[lock['rse_id']][did_key] = {'length': lock['length'] or 0, 'bytes': lock['bytes'] or 0}
+                if no_locks:
+                    logger(logging.WARNING, 'Cannot find a lock for rule %s on DID %s', rule.id, did)
+                if not dry_run:
+                    lifetime = grace_period + rand.randrange(spread_period + 1)
+                    logger(logging.INFO, 'Setting %s seconds lifetime for rule %s', lifetime, rule.id)
+                    options = {'lifetime': lifetime}
+                    if purge_replicas:
+                        options['purge_replicas'] = True
+                    if rule.locked and unlock:
+                        logger(logging.INFO, 'Unlocking rule %s', rule.id)
+                        options['locked'] = False
+                    try:
+                        update_rule(rule.id, options=options)
+                    except RuleNotFound:
+                        logger(logging.WARNING, 'Cannot find rule %s on DID %s', rule.id, did)
+                        continue
+    except Exception:
+        exc_type, exc_value, exc_traceback = exc_info()
+        logger(logging.CRITICAL, ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())
+
+    for rse_id in summary:
+        tot_size, tot_files, tot_datasets = 0, 0, 0
+        for did in summary[rse_id]:
+            tot_datasets += 1
+            tot_files += summary[rse_id][did].get('length', 0)
+            tot_size += summary[rse_id][did].get('bytes', 0)
+        vo = get_rse_vo(rse_id=rse_id)
+        logger(logging.INFO, 'For RSE %s%s %d datasets will be deleted representing %d files and %d bytes',
+               get_rse_name(rse_id=rse_id), '' if vo == 'def' else ' on VO ' + vo, tot_datasets, tot_files, tot_size)
+
+
+def run(
+        date_check: datetime.datetime = datetime.datetime.utcnow(),
+        dry_run: bool = True,
+        grace_period: int = 86400,
+        purge_replicas: bool = False,
+        spread_period: int = 0,
+        unlock: bool = False,
+        once: bool = True,
+        sleep_time: int = 60,
+        threads: int = 1,
+) -> None:
+    """
+    Starts up the atropos threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('Will run only one iteration')
+    logging.info('starting atropos threads')
+    thread_list = [threading.Thread(target=atropos, kwargs={'date_check': date_check,
+                                                            'dry_run': dry_run,
+                                                            'grace_period': grace_period,
+                                                            'purge_replicas': purge_replicas,
+                                                            'spread_period': spread_period,
+                                                            'unlock': unlock,
+                                                            'once': once,
+                                                            'sleep_time': sleep_time}) for i in range(0, threads)]
+    [t.start() for t in thread_list]
+
+    logging.info('waiting for interrupts')
+
+    # Interruptible joins require a timeout.
+    while thread_list:
+        thread_list = [t.join(timeout=3.14) for t in thread_list if t and t.is_alive()]
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    GRACEFUL_STOP.set()

rucio/daemons/auditor/__init__.py

@@ -0,0 +1,289 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import bz2
+import glob
+import logging
+import os
+import select
+from datetime import datetime, timedelta
+from queue import Empty as EmptyQueue
+from typing import TYPE_CHECKING, Optional
+
+from rucio.common import config
+from rucio.common.dumper import LogPipeHandler, mkdir, temp_file
+from rucio.common.dumper.consistency import Consistency
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.utils import chunks
+from rucio.core.quarantined_replica import add_quarantined_replicas
+from rucio.core.replica import declare_bad_file_replicas, list_replicas
+from rucio.core.rse import get_rse_id, get_rse_usage
+from rucio.daemons.auditor import srmdumps
+from rucio.daemons.auditor.hdfs import ReplicaFromHDFS
+from rucio.db.sqla.constants import BadFilesStatus
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from configparser import RawConfigParser
+    from multiprocessing import Queue as QueueType
+    from multiprocessing.connection import Connection
+    from multiprocessing.synchronize import Event
+
+
+def consistency(
+        rse: str,
+        delta: timedelta,
+        configuration: "RawConfigParser",
+        cache_dir: str,
+        results_dir: str
+) -> Optional[str]:
+    logger = logging.getLogger('auditor-worker')
+    rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
+    results_path = os.path.join(results_dir, '{0}_{1}'.format(rse, rsedate.strftime('%Y%m%d')))  # pylint: disable=no-member
+
+    if os.path.exists(results_path + '.bz2') or os.path.exists(results_path):
+        logger.warning('Consistency check for "%s" (dump dated %s) already done, skipping check', rse, rsedate.strftime('%Y%m%d'))  # pylint: disable=no-member
+        return None
+
+    rrdump_prev = ReplicaFromHDFS.download(rse, rsedate - delta, cache_dir=cache_dir)
+    rrdump_next = ReplicaFromHDFS.download(rse, rsedate + delta, cache_dir=cache_dir)
+    results = Consistency.dump(
+        'consistency-manual',
+        rse,
+        rsedump,
+        rrdump_prev,
+        rrdump_next,
+        date=rsedate,
+        cache_dir=cache_dir,
+    )
+    mkdir(results_dir)
+    with temp_file(results_dir, results_path) as (output, _):
+        for result in results:
+            output.write('{0}\n'.format(result.csv()))
+
+    return results_path
+
+
+def guess_replica_info(
+        path: str
+) -> tuple[Optional[str], str]:
+    """Try to extract the scope and name from a path.
+
+    ``path``: relative path to the file on the RSE.
+
+    Returns a ``tuple`` of which the first element is the scope of the
+    replica and the second element is its name.
+    """
+    items = path.split('/')
+    if len(items) == 1:
+        return None, path
+    elif len(items) > 2 and items[0] in ['group', 'user']:
+        return '.'.join(items[0:2]), items[-1]
+    else:
+        return items[0], items[-1]
+
+
+def bz2_compress_file(
+        source: str,
+        chunk_size: int = 65000
+) -> str:
+    """Compress a file with bzip2.
+
+    The destination is the path passed through ``source`` extended with
+    '.bz2'.  The original file is deleted.
+
+    Errors are deliberately not handled gracefully.  Any exceptions
+    should be propagated to the caller.
+
+    ``source``: absolute path to the file to compress.
+
+    ``chunk_size``: size (in bytes) of the chunks by which to read the file.
+
+    Returns the destination path.
+    """
+    destination = '{}.bz2'.format(source)
+    with open(source) as plain, bz2.BZ2File(destination, 'w') as compressed:
+        while True:
+            chunk = plain.read(chunk_size)
+            if not chunk:
+                break
+            compressed.write(chunk.encode())
+    os.remove(source)
+    return destination
+
+
+def process_output(
+        output: str,
+        sanity_check: bool = True,
+        compress: bool = True
+) -> None:
+    """Perform post-consistency-check actions.
+
+    DARK files are put in the quarantined-replica table so that they
+    may be deleted by the Dark Reaper.  LOST files are reported as
+    suspicious so that they may be further checked by the cloud squads.
+
+    ``output``: absolute path to the file
+    produced by ``consistency()``.  It must maintain its naming
+    convention.
+
+    If ``sanity_check`` is ``True`` (default) and the number of entries
+    in the output file is deemed excessive, the actions are aborted.
+
+    If ``compress`` is ``True`` (default), the file is compressed with
+    bzip2 after the actions are successfully performed.
+    """
+    logger = logging.getLogger('auditor-worker')
+    dark_replicas = []
+    lost_replicas = []
+    try:
+        with open(output) as f:
+            for line in f:
+                label, path = line.rstrip().split(',', 1)
+                scope, name = guess_replica_info(path)
+                if label == 'DARK':
+                    dark_replicas.append({'path': path,
+                                          'scope': InternalScope(scope),
+                                          'name': name})
+                elif label == 'LOST':
+                    lost_replicas.append({'scope': InternalScope(scope),
+                                          'name': name})
+                else:
+                    raise ValueError('unexpected label')
+    # Since the file is read immediately after its creation, any error
+    # exposes a bug in the Auditor.
+    except Exception as error:
+        logger.critical('Error processing "%s"', output, exc_info=True)
+        raise error
+
+    rse = os.path.basename(output[:output.rfind('_')])
+    rse_id = get_rse_id(rse=rse)
+    usage = get_rse_usage(rse_id=rse_id, source='rucio')[0]
+    threshold = config.config_get_float('auditor', 'threshold', False, 0.1)
+
+    # Perform a basic sanity check by comparing the number of entries
+    # with the total number of files on the RSE.  If the percentage is
+    # significant, there is most likely an issue with the site dump.
+    found_error = False
+    if len(dark_replicas) > threshold * usage['files']:
+        logger.warning('Number of DARK files is exceeding threshold: "%s"',
+                       output)
+        found_error = True
+    if len(lost_replicas) > threshold * usage['files']:
+        logger.warning('Number of LOST files is exceeding threshold: "%s"',
+                       output)
+        found_error = True
+    if found_error and sanity_check:
+        raise AssertionError('sanity check failed')
+
+    # While converting LOST replicas to PFNs, entries that do not
+    # correspond to a replica registered in Rucio are silently dropped.
+    lost_pfns = [r['rses'][rse_id][0] for chunk in chunks(lost_replicas, 1000) for r in list_replicas(chunk) if rse_id in r['rses']]
+
+    for chunk in chunks(dark_replicas, 1000):
+        add_quarantined_replicas(rse_id=rse_id, replicas=chunk)
+    logger.debug('Processed %d DARK files from "%s"', len(dark_replicas),
+                 output)
+    declare_bad_file_replicas(lost_pfns, reason='Reported by Auditor',
+                              issuer=InternalAccount('root'), status=BadFilesStatus.SUSPICIOUS)
+    logger.debug('Processed %d LOST files from "%s"', len(lost_replicas),
+                 output)
+
+    if compress:
+        destination = bz2_compress_file(output)
+        logger.debug('Compressed "%s"', destination)
+
+
+def check(
+        queue: "QueueType",
+        retry: "QueueType",
+        terminate: "Event",
+        logpipe: "Connection",
+        cache_dir: str,
+        results_dir: str,
+        keep_dumps: bool,
+        delta_in_days: int
+) -> None:
+    logger = logging.getLogger('auditor-worker')
+    lib_logger = logging.getLogger('dumper')
+
+    loglevel = logging.getLevelName(config.config_get('common', 'loglevel', False, 'DEBUG'))
+    logger.setLevel(loglevel)
+    lib_logger.setLevel(loglevel)
+
+    handler = LogPipeHandler(logpipe)
+    logger.addHandler(handler)
+    lib_logger.addHandler(handler)
+
+    formatter = logging.Formatter(
+        "%(asctime)s  %(name)-22s  %(levelname)-8s [PID %(process)8d] %(message)s"
+    )
+    handler.setFormatter(formatter)
+
+    delta = timedelta(days=delta_in_days)
+
+    configuration = srmdumps.parse_configuration()
+
+    while not terminate.is_set():
+        try:
+            rse, attempts = queue.get(timeout=30)
+        except EmptyQueue:
+            continue
+        start = datetime.now()
+        try:
+            logger.debug('Checking "%s"', rse)
+            output = consistency(rse, delta, configuration, cache_dir,
+                                 results_dir)
+            if output:
+                process_output(output)
+        except:
+            elapsed = (datetime.now() - start).total_seconds() / 60
+            logger.error('Check of "%s" failed in %d minutes, %d remaining attempts', rse, elapsed, attempts, exc_info=True)
+            success = False
+        else:
+            elapsed = (datetime.now() - start).total_seconds() / 60
+            logger.info('SUCCESS checking "%s" in %d minutes', rse, elapsed)
+            success = True
+
+        if not keep_dumps:
+            remove = glob.glob(os.path.join(cache_dir, 'replicafromhdfs_{0}_*'.format(rse)))
+            remove.extend(glob.glob(os.path.join(cache_dir, 'ddmendpoint_{0}_*'.format(rse))))
+            logger.debug('Removing: %s', remove)
+            for fil in remove:
+                os.remove(fil)
+
+        if not success and attempts > 0:
+            retry.put((rse, attempts - 1))
+
+
+def activity_logger(
+        logpipes: "Iterable[Connection]",
+        logfilename: str,
+        terminate: "Event"
+) -> None:
+    handler = logging.handlers.RotatingFileHandler(
+        logfilename,
+        maxBytes=20971520,
+        backupCount=10,
+    )
+    handler.setFormatter(logging.Formatter(fmt=None))
+    logger = logging.getLogger('auditor-logger-raw')
+    logger.addHandler(handler)
+    logger.setLevel(logging.CRITICAL)  # The level of this logger is irrelevant
+
+    while not terminate.is_set():
+        ready, _, _ = select.select(logpipes, tuple(), tuple(), 30)
+        if ready:
+            for logpipe in ready:
+                logger.critical(logpipe.recv())