rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/oauthmanager/oauthmanager.py

@@ -0,0 +1,198 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+OAuth Manager is a daemon which is responsible for:
+- deletion of expired access tokens (in case there is a valid refresh token,
+  expired access tokens will be kept until refresh_token expires as well.)
+- deletion of expired OAuth session parameters
+- refreshing access tokens via their refresh tokens.
+
+These 3 actions run consequently one after another in a loop with a sleeptime of 'looprate' seconds.
+The maximum number of DB rows (tokens, parameters, refresh tokens) on which the script will operate
+can be specified by 'max_rows' parameter.
+
+"""
+
+import functools
+import logging
+import threading
+import traceback
+from re import match
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common.exception import DatabaseException
+from rucio.common.logging import setup_logging
+from rucio.common.stopwatch import Stopwatch
+from rucio.core.authentication import delete_expired_tokens
+from rucio.core.monitor import MetricManager
+from rucio.core.oidc import delete_expired_oauthrequests, refresh_jwt_tokens
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+METRICS = MetricManager(module=__name__)
+graceful_stop = threading.Event()
+DAEMON_NAME = 'oauth-manager'
+
+
+def oauth_manager(once: bool = False, max_rows: int = 100, sleep_time: int = 300) -> None:
+    """
+    Main loop to delete all expired tokens, refresh tokens eligible
+    for refresh and delete all expired OAuth session parameters.
+    It was decided to have only 1 daemon for all 3 of these cleanup activities.
+
+    :param once: If True, the loop is run just once, otherwise the daemon continues looping until stopped.
+    :param max_rows: Max number of DB rows to deal with per operation.
+    :param sleep_time: The number of seconds the daemon will wait before running next loop of operations.
+
+    :returns: None
+    """
+
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            max_rows=max_rows,
+            sleep_time=sleep_time
+        ),
+    )
+
+
+def run_once(heartbeat_handler: HeartbeatHandler, max_rows: int, sleep_time: int, **_kwargs) -> None:
+
+    # make an initial heartbeat
+    heartbeat_handler.live()
+
+    stopwatch = Stopwatch()
+
+    ndeleted = 0
+    ndeletedreq = 0
+    nrefreshed = 0
+
+    # make a heartbeat
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    try:
+        # ACCESS TOKEN REFRESH - better to run first (in case some of the refreshed tokens needed deletion after this step)
+        logger(logging.INFO, '----- START ----- ACCESS TOKEN REFRESH ----- ')
+        logger(logging.INFO, 'starting to query tokens for automatic refresh')
+        nrefreshed = refresh_jwt_tokens(total_workers, worker_number, refreshrate=int(sleep_time), limit=max_rows)
+        logger(logging.INFO, 'successfully refreshed %i tokens', nrefreshed)
+        logger(logging.INFO, '----- END ----- ACCESS TOKEN REFRESH ----- ')
+        METRICS.counter(name='oauth_manager.tokens.refreshed').inc(nrefreshed)
+
+    except (DatabaseException, DatabaseError) as err:
+        if match('.*QueuePool.*', str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        else:
+            logger(logging.CRITICAL, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+
+    try:
+        # waiting 1 sec as DBs does not store millisecond and tokens
+        # eligible for deletion after refresh might not get deleted otherwise
+        graceful_stop.wait(1)
+
+        # make a heartbeat
+        worker_number, total_workers, logger = heartbeat_handler.live()
+
+        # EXPIRED TOKEN DELETION
+        logger(logging.INFO, '----- START ----- DELETION OF EXPIRED TOKENS ----- ')
+        logger(logging.INFO, 'starting to delete expired tokens')
+        ndeleted += delete_expired_tokens(total_workers, worker_number, limit=max_rows)
+        logger(logging.INFO, 'deleted %i expired tokens', ndeleted)
+        logger(logging.INFO, '----- END ----- DELETION OF EXPIRED TOKENS ----- ')
+        METRICS.counter(name='oauth_manager.tokens.deleted').inc(ndeleted)
+
+    except (DatabaseException, DatabaseError) as err:
+        if match('.*QueuePool.*', str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        else:
+            logger(logging.CRITICAL, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+
+    try:
+        # make a heartbeat
+        worker_number, total_workers, logger = heartbeat_handler.live()
+
+        # DELETING EXPIRED OAUTH SESSION PARAMETERS
+        logger(logging.INFO, '----- START ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ')
+        logger(logging.INFO, 'starting deletion of expired OAuth session requests')
+        ndeletedreq += delete_expired_oauthrequests(total_workers, worker_number, limit=max_rows)
+        logger(logging.INFO, 'expired parameters of %i authentication requests were deleted', ndeletedreq)
+        logger(logging.INFO, '----- END ----- DELETION OF EXPIRED OAUTH SESSION REQUESTS ----- ')
+        METRICS.counter(name='oauth_manager.oauthreq.deleted').inc(ndeletedreq)
+
+    except (DatabaseException, DatabaseError) as err:
+        if match('.*QueuePool.*', str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
+            logger(logging.WARNING, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+        else:
+            logger(logging.CRITICAL, traceback.format_exc())
+            METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
+
+    stopwatch.stop()
+    logger(logging.INFO, 'took %f seconds to delete %i tokens, %i session parameters and refreshed %i tokens', stopwatch.elapsed, ndeleted, ndeletedreq, nrefreshed)
+    METRICS.timer('duration').observe(stopwatch.elapsed)
+
+
+def run(once: bool = False, threads: int = 1, max_rows: int = 100, sleep_time: int = 300) -> None:
+    """
+    Starts up the OAuth Manager threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        oauth_manager(once, max_rows, sleep_time)
+    else:
+        logging.info('OAuth Manager starting %s threads', str(threads))
+        threads = [threading.Thread(target=oauth_manager,
+                                    kwargs={'once': once,
+                                            'max_rows': max_rows,
+                                            'sleep_time': sleep_time}) for i in range(0, threads)]
+        _ = [t.start() for t in threads]
+        # Interruptible joins require a timeout.
+        while threads[0].is_alive():
+            _ = [t.join(timeout=3.14) for t in threads]
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()

rucio/daemons/reaper/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/reaper/dark_reaper.py

@@ -0,0 +1,282 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Dark Reaper is a daemon to manage quarantined file deletion.
+'''
+
+import functools
+import logging
+import random
+import sys
+import threading
+import time
+import traceback
+from typing import TYPE_CHECKING
+
+import rucio.core.rse as rse_core
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.config import config_get_bool
+from rucio.common.exception import ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, ServiceUnavailable, SourceNotFound, VONotFound
+from rucio.common.logging import setup_logging
+from rucio.core.message import add_message
+from rucio.core.monitor import MetricManager
+from rucio.core.quarantined_replica import delete_quarantined_replicas, list_quarantined_replicas, list_rses_with_quarantined_replicas
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.core.vo import list_vos
+from rucio.daemons.common import run_daemon
+from rucio.rse import rsemanager as rsemgr
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from types import FrameType
+    from typing import Optional
+
+    from rucio.common.types import LFNDict
+    from rucio.daemons.common import HeartbeatHandler
+
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'dark-reaper'
+
+
+def reaper(
+        rses: "Sequence[str]",
+        chunk_size: int = 100,
+        once: bool = False,
+        scheme: "Optional[str]" = None,
+        sleep_time: int = 300,
+):
+    executable = DAEMON_NAME
+    if rses:
+        executable += ' '.join(sys.argv[1:])
+    run_daemon(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=executable,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            rses=rses,
+            chunk_size=chunk_size,
+            scheme=scheme,
+        ),
+    )
+
+
+def run_once(
+        rses: "Sequence[str]",
+        heartbeat_handler: "HeartbeatHandler",
+        chunk_size: int = 100,
+        scheme: "Optional[str]" = None,
+        **_kwargs,
+):
+    """
+    Main loop to select and delete files.
+
+    :param rses: List of RSEs the reaper should work against.
+    :param chunk_size: the size of chunk for deletion.
+    :param scheme: Force the reaper to use a particular protocol, e.g., mock.
+    """
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.INFO, 'Starting Dark Reaper on RSEs: %s', ', '.join(rses))
+
+    nothing_to_do = True
+    rses_to_process = list(set(rses) & set(list_rses_with_quarantined_replicas()))
+    random.shuffle(rses_to_process)
+    for rse_id in rses_to_process:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        # The following query returns the list of real replicas (deleted_replicas) and list of dark replicas (dark_replicas)
+        # Real replicas can be directly removed from the quarantine table
+        deleted_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
+                                                                    limit=chunk_size,
+                                                                    worker_number=worker_number,
+                                                                    total_workers=total_workers)
+
+        rse_info = rsemgr.get_rse_info(rse_id=rse_id)
+        rse = rse_info['rse']
+        prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme)
+
+        try:
+            prot.connect()
+            for replica in dark_replicas:
+                worker_number, total_workers, logger = heartbeat_handler.live()
+                nothing_to_do = False
+                scope = ''
+                if replica['scope']:
+                    scope = replica['scope'].external
+                try:
+                    lfn: "LFNDict" = {
+                        'scope': scope,
+                        'name': replica['name'],
+                        'path': replica['path']
+                    }
+                    pfn = str(list(rsemgr.lfns2pfns(rse_settings=rse_info,
+                                                    lfns=[lfn],
+                                                    operation='delete',
+                                                    scheme=scheme).values())[0])
+                    logger(logging.INFO, 'Deletion ATTEMPT of %s:%s as %s on %s', scope, replica['name'], pfn, rse)
+                    start = time.time()
+                    prot.delete(pfn)
+                    METRICS.counter('deleted_replicas').inc()
+                    duration = time.time() - start
+                    logger(logging.INFO, 'Deletion SUCCESS of %s:%s as %s on %s in %s seconds', scope, replica['name'], pfn, rse, duration)
+                    payload = {'scope': scope,
+                               'name': replica['name'],
+                               'rse': rse,
+                               'rse_id': rse_id,
+                               'file-size': replica.get('bytes') or 0,
+                               'bytes': replica.get('bytes') or 0,
+                               'url': pfn,
+                               'duration': duration,
+                               'protocol': prot.attributes['scheme']}
+                    if replica['scope'].vo != 'def':
+                        payload['vo'] = replica['scope'].vo
+                    add_message('deletion-done', payload)
+                    deleted_replicas.append(replica)
+                except SourceNotFound:
+                    err_msg = ('Deletion NOTFOUND of %s:%s as %s on %s'
+                               % (scope, replica['name'], pfn, rse))
+                    logger(logging.WARNING, err_msg)
+                    deleted_replicas.append(replica)
+                except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error:
+                    err_msg = ('Deletion NOACCESS of %s:%s as %s on %s: %s'
+                               % (scope, replica['name'], pfn, rse, str(error)))
+                    logger(logging.WARNING, err_msg)
+                    payload = {'scope': scope,
+                               'name': replica['name'],
+                               'rse': rse,
+                               'rse_id': rse_id,
+                               'file-size': replica['bytes'] or 0,
+                               'bytes': replica['bytes'] or 0,
+                               'url': pfn,
+                               'reason': str(error),
+                               'protocol': prot.attributes['scheme']}
+                    if replica['scope'].vo != 'def':
+                        payload['vo'] = replica['scope'].vo
+                    add_message('deletion-failed', payload)
+
+                except Exception:
+                    logging.critical(traceback.format_exc())
+        finally:
+            prot.close()
+
+        delete_quarantined_replicas(rse_id=rse_id, replicas=deleted_replicas)
+
+    must_sleep = False
+    if nothing_to_do:
+        logger(logging.INFO, 'Nothing to do')
+        must_sleep = True
+    return must_sleep
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    GRACEFUL_STOP.set()
+
+
+def run(
+    total_workers: int = 1,
+    chunk_size: int = 100,
+    once: bool = False,
+    rses: "Optional[list[str]]" = None,
+    scheme: "Optional[str]" = None,
+    exclude_rses: "Optional[str]" = None,
+    include_rses: "Optional[str]" = None,
+    vos: "Optional[list[str]]" = None,
+    delay_seconds: int = 0,
+    sleep_time: int = 300
+) -> None:
+    """
+    Starts up the reaper threads.
+
+    :param total_workers: The total number of workers.
+    :param chunk_size: the size of chunk for deletion.
+    :param once: If True, only runs one iteration of the main loop.
+    :param rses: List of RSEs the reaper should work against. If empty, it considers all RSEs (Single-VO only).
+    :param scheme: Force the reaper to use a particular protocol/scheme, e.g., mock.
+    :param exclude_rses: RSE expression to exclude RSEs from the Reaper.
+    :param include_rses: RSE expression to include RSEs.
+    :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
+                If None, we either use all VOs if run from "def", or the current VO otherwise.
+    """
+    rses = rses or []
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    logging.info('main: starting processes')
+
+    multi_vo = config_get_bool('common', 'multi_vo', raise_exception=False, default=False)
+    if not multi_vo:
+        if vos:
+            logging.warning('Ignoring argument vos, this is only applicable in a multi-VO setup.')
+        vos = ['def']
+    else:
+        if vos:
+            invalid = set(vos) - set([v['vo'] for v in list_vos()])
+            if invalid:
+                msg = 'VO{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
+                                                       ', '.join([repr(v) for v in invalid]))
+                raise VONotFound(msg)
+        else:
+            vos = [v['vo'] for v in list_vos()]
+        logging.info('Dark Reaper: This instance will work on VO%s: %s'
+                     % ('s' if len(vos) > 1 else '', ', '.join([v for v in vos])))
+
+    all_rses = []
+    for vo in vos:
+        all_rses.extend([rse['id'] for rse in rse_core.list_rses(filters={'vo': vo})])
+
+    if rses:
+        invalid = set(rses) - set([rse['rse'] for rse in all_rses])
+        if invalid:
+            msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
+                                                    ', '.join([repr(rse) for rse in invalid]))
+            raise RSENotFound(msg)
+        rses = [rse for rse in all_rses if rse['rse'] in rses]
+    else:
+        rses = all_rses
+
+    if exclude_rses:
+        excluded_rses = [rse['id'] for rse in parse_expression(exclude_rses)]
+        rses = [rse for rse in rses if rse not in excluded_rses]
+
+    if include_rses:
+        included_rses = [rse['id'] for rse in parse_expression(include_rses)]
+        rses = [rse for rse in rses if rse in included_rses]
+
+    if not rses:
+        logging.error('Dark Reaper: No RSEs found. Exiting.')
+        return
+
+    threads = []
+    for worker in range(total_workers):
+        kwargs = {'rses': rses,
+                  'once': once,
+                  'chunk_size': chunk_size,
+                  'scheme': scheme,
+                  'sleep_time': sleep_time}
+        threads.append(threading.Thread(target=reaper, kwargs=kwargs,
+                                        name='Worker: %s, Total_Workers: %s' % (worker, total_workers)))
+    [t.start() for t in threads]
+    while threads[0].is_alive():
+        [t.join(timeout=3.14) for t in threads]