rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/cache/consumer.py

@@ -21,13 +21,12 @@ import logging
 import threading
 import time
 from traceback import format_exc
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
 from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.stomp_utils import StompConnectionManager
+from rucio.common.stomp_utils import ListenerBase, StompConnectionManager
 from rucio.common.types import InternalScope, LoggerFunction
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import get_rse_id
@@ -36,7 +35,6 @@ from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_r
 if TYPE_CHECKING:
     from types import FrameType
 
-    from stomp import Connection
     from stomp.utils import Frame
 
 logging.getLogger("stomp").setLevel(logging.CRITICAL)
@@ -46,35 +44,11 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'cache-consumer'
 
 
-class AMQConsumer:
+class AMQConsumer(ListenerBase):
     """
     class Consumer
     """
 
-    def __init__(
-            self,
-            broker: str,
-            conn: "Connection",
-            logger: "LoggerFunction"
-    ):
-        """
-        __init__
-        """
-        self.__broker = broker
-        self.__conn = conn
-        self.__logger = logger
-
-    @METRICS.count_it
-    def on_heartbeat_timeout(self) -> None:
-        self.__conn.disconnect()
-
-    @METRICS.count_it
-    def on_error(self, frame: "Frame") -> None:
-        """
-        on_error
-        """
-        self.__logger(logging.ERROR, 'Message receive error: [%s] %s' % (self.__broker, frame.body))
-
     @METRICS.count_it
     def on_message(self, frame: "Frame") -> None:
         """
@@ -82,7 +56,7 @@ class AMQConsumer:
         """
         try:
             msg = json.loads(frame.body)  # type: ignore
-            self.__logger(logging.DEBUG, 'Message received: %s ' % msg)
+            self._logger(logging.DEBUG, 'Message received: %s', msg)
             if isinstance(msg, dict) and 'operation' in msg.keys():
                 for f in msg['files']:
                     f['scope'] = InternalScope(f['scope'])
@@ -93,81 +67,42 @@ class AMQConsumer:
 
                 rse_vo_str = msg['rse']
                 if 'vo' in msg and msg['vo'] != 'def':
-                    rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo'])
+                    rse_vo_str = f"{rse_vo_str} on {msg['vo']}"
                 if msg['operation'] == 'add_replicas':
-                    self.__logger(logging.INFO, 'add_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    self._logger(logging.INFO, "add_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
                     add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
                 elif msg['operation'] == 'delete_replicas':
-                    self.__logger(logging.INFO, 'delete_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    self._logger(logging.INFO, "delete_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
                     delete_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
             else:
-                self.__logger(logging.DEBUG, 'Check failed: %s %s '
-                              % (isinstance(msg, dict), 'operation' in msg.keys()))
+                self._logger(logging.DEBUG, 'Check failed: %s %s', isinstance(msg, dict), "operation" in msg.keys())
         except:
-            self.__logger(logging.ERROR, str(format_exc()))
+            self._logger(logging.ERROR, str(format_exc()))
 
 
-def consumer(id_: int, num_thread: int = 1) -> None:
+def consumer(id_: int, num_thread: int = 1, logger: LoggerFunction = logging.log) -> None:
     """
     Main loop to consume messages from the Rucio Cache producer.
     """
-
-    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
-
     logger(logging.INFO, 'Rucio Cache consumer starting')
 
-    brokers = config_get_list('messaging-cache', 'brokers')
-
-    use_ssl = config_get_bool('messaging-cache', 'use_ssl', default=True, raise_exception=False)
-    if not use_ssl:
-        username = config_get('messaging-cache', 'username')
-        password = config_get('messaging-cache', 'password')
-    destination = config_get('messaging-cache', 'destination')
-    subscription_id = 'rucio-cache-messaging'
-
-    vhost = config_get('messaging-cache', 'broker_virtual_host', raise_exception=False)
-    port = config_get_int('messaging-cache', 'port')
-    reconnect_attempts = config_get_int('messaging-cache', 'reconnect_attempts', default=100)
-    ssl_key_file = config_get('messaging-cache', 'ssl_key_file', raise_exception=False)
-    ssl_cert_file = config_get('messaging-cache', 'ssl_cert_file', raise_exception=False)
-
-    stomp_conn_mngr = StompConnectionManager()
-    conns, _ = stomp_conn_mngr.re_configure(
-        brokers=brokers,
-        port=port,
-        use_ssl=use_ssl,
-        vhost=vhost,
-        reconnect_attempts=reconnect_attempts,
-        ssl_key_file=ssl_key_file,
-        ssl_cert_file=ssl_cert_file,
-        timeout=None,
-        logger=logger
-    )
+    conn_mgr = StompConnectionManager(config_section='messaging-cache', logger=logger)
 
     logger(logging.INFO, 'consumer started')
 
+    conn_mgr.set_listener_factory('rucio-cache-consumer', AMQConsumer, heartbeats=conn_mgr.config.heartbeats)
+
     while not GRACEFUL_STOP.is_set():
-        for conn in conns:
-            if not conn.is_connected():
-                host_port = conn.transport._Transport__host_and_ports[0]
-
-                logger(logging.INFO, 'connecting to %s' % host_port[0])
-                METRICS.counter('reconnect.{host}').labels(host=host_port[0]).inc()
-                conn.set_listener('rucio-cache-consumer', AMQConsumer(broker=host_port, conn=conn, logger=logger))
-                if not use_ssl:
-                    conn.connect(username, password)
-                else:
-                    conn.connect()
 
-                conn.subscribe(destination=destination, ack='auto', id=subscription_id)
+        conn_mgr.subscribe(id_='rucio-cache-messaging', ack='auto')
         time.sleep(1)
 
     logger(logging.INFO, 'graceful stop requested')
-    stomp_conn_mngr.disconnect()
+    conn_mgr.disconnect()
     logger(logging.INFO, 'graceful stop done')
 
 
-def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
     """
     Graceful exit.
     """
@@ -180,18 +115,19 @@ def run(num_thread: int = 1) -> None:
     Starts up the rucio cache consumer thread
     """
     setup_logging(process_name=DAEMON_NAME)
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
 
     if rucio.db.sqla.util.is_old_db():
         raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
-    logging.info('starting consumer thread')
-    threads = [threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread})
-               for i in range(0, num_thread)]
-
-    [t.start() for t in threads]
+    logger(logging.INFO, 'starting consumer thread')
+    threads = []
+    for i in range(num_thread):
+        con_thread = threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread, 'logger': logger})
+        con_thread.start()
+        threads.append(con_thread)
 
-    logging.info('waiting for interrupts')
+    logger(logging.INFO, 'waiting for interrupts')
 
-    # Interruptible joins require a timeout.
-    while threads[0].is_alive():
-        [t.join(timeout=3.14) for t in threads]
+    while [thread.join(timeout=3.14) for thread in threads if thread.is_alive()]:
+        pass

rucio/daemons/common.py

@@ -20,7 +20,6 @@ import queue
 import socket
 import threading
 import time
-from collections.abc import Callable, Generator, Iterator, Sequence
 from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union
 
 from rucio.common.logging import formatted_logger
@@ -29,6 +28,8 @@ from rucio.core import heartbeat as heartbeat_core
 from rucio.core.monitor import MetricManager
 
 if TYPE_CHECKING:
+    from collections.abc import Callable, Generator, Iterator, Sequence
+
     from rucio.common.types import LoggerFunction
 
 T = TypeVar('T')
@@ -85,7 +86,7 @@ class HeartbeatHandler:
             self,
             force_renew: bool = False,
             payload: Optional[str] = None
-    ) -> tuple[int, int, Callable]:
+    ) -> tuple[int, int, 'Callable']:
         """
         :return: a tuple: <the number of the current worker>, <total number of workers>, <decorated logger>
         """
@@ -115,9 +116,9 @@ class HeartbeatHandler:
 def _activity_looper(
         once: bool,
         sleep_time: int,
-        activities: Optional[Sequence[str]],
+        activities: Optional['Sequence[str]'],
         heartbeat_handler: HeartbeatHandler,
-) -> Generator[tuple[str, float], tuple[float, bool], None]:
+) -> 'Generator[tuple[str, float], tuple[float, bool], None]':
     """
     Generator which loops (either once, or indefinitely) over all activities while ensuring that `sleep_time`
     passes between handling twice the same activity.
@@ -164,7 +165,7 @@ def _activity_looper(
         if not once:
             if must_sleep:
                 time_diff = time.time() - actual_exe_time
-                time_to_sleep = max(1, sleep_time - time_diff)
+                time_to_sleep = max(1.0, sleep_time - time_diff)
                 activity_next_exe_time[activity] = time.time() + time_to_sleep
             else:
                 activity_next_exe_time[activity] = time.time() + 1
@@ -176,19 +177,8 @@ def db_workqueue(
         executable: str,
         partition_wait_time: int,
         sleep_time: int,
-        activities: Optional[Sequence[str]] = None,
-) -> Callable[
-    [
-        Callable[
-            ...,
-            Union[bool, tuple[bool, T], None]
-        ]
-    ],
-    Callable[
-        [],
-        Iterator[Union[T, None]]
-    ]
-]:
+        activities: Optional['Sequence[str]'] = None,
+) -> 'Callable[[Callable[..., Union[bool, tuple[bool, T], None]]], Callable[[], Iterator[Union[T, None]]]]':
     """
     Used to wrap a function for interacting with the database as a work queue: i.e. to select
     a set of rows and perform some work on those rows while ensuring that two instances running in parallel don't
@@ -203,10 +193,10 @@ def db_workqueue(
     :param activities: optional list of activities on which to work. The run_once_fnc will be called on activities one by one.
     """
 
-    def _decorate(run_once_fnc: Callable[..., Optional[Union[bool, tuple[bool, T]]]]) -> Callable[[], Iterator[Optional[T]]]:
+    def _decorate(run_once_fnc: 'Callable[..., Optional[Union[bool, tuple[bool, T]]]]') -> 'Callable[[], Iterator[Optional[T]]]':
 
         @functools.wraps(run_once_fnc)
-        def _generator() -> Iterator[T]:
+        def _generator() -> 'Iterator[T]':
 
             with HeartbeatHandler(executable=executable, renewal_interval=sleep_time - 1) as heartbeat_handler:
                 logger = heartbeat_handler.logger
@@ -270,7 +260,7 @@ def run_daemon(
         executable: str,
         partition_wait_time: int,
         sleep_time: int,
-        run_once_fnc: Callable[..., Optional[Union[bool, tuple[bool, Any]]]],
+        run_once_fnc: 'Callable[..., Optional[Union[bool, tuple[bool, Any]]]]',
         activities: Optional[list[str]] = None
 ) -> None:
     """
@@ -297,8 +287,8 @@ class ProducerConsumerDaemon(Generic[T]):
 
     def __init__(
             self,
-            producers: Sequence[Callable[[], Iterator[T]]],
-            consumers: Sequence[Callable[..., None]],
+            producers: 'Sequence[Callable[[], Iterator[T]]]',
+            consumers: 'Sequence[Callable[..., None]]',
             graceful_stop: threading.Event,
             logger: "LoggerFunction" = logging.log
     ):
@@ -314,7 +304,7 @@ class ProducerConsumerDaemon(Generic[T]):
 
     def _produce(
             self,
-            it: Callable[[], Iterator[T]],
+            it: 'Callable[[], Iterator[T]]',
             wait_for_consumers: bool = False
     ) -> None:
         """
@@ -351,7 +341,7 @@ class ProducerConsumerDaemon(Generic[T]):
 
     def _consume(
             self,
-            fnc: Callable[[T], Any]
+            fnc: 'Callable[[T], Any]'
     ) -> None:
         """
         Wait for elements to arrive via the queue and call the given function on each element.

rucio/daemons/conveyor/finisher.py

@@ -29,7 +29,7 @@ from dogpile.cache.api import NoValue
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
-from rucio.common.cache import make_region_memcached
+from rucio.common.cache import MemcacheRegion
 from rucio.common.config import config_get_bool, config_get_list
 from rucio.common.exception import DatabaseException, ReplicaNotFound, RequestNotFound, RSEProtocolNotSupported, UnsupportedOperation
 from rucio.common.logging import setup_logging
@@ -55,7 +55,7 @@ if TYPE_CHECKING:
 
 GRACEFUL_STOP = threading.Event()
 
-REGION = make_region_memcached(expiration_time=900)
+REGION = MemcacheRegion(expiration_time=900)
 METRICS = MetricManager(module=__name__)
 DAEMON_NAME = 'conveyor-finisher'
 FAILED_DURING_SUBMISSION_DELAY = datetime.timedelta(minutes=120)

rucio/daemons/conveyor/poller.py

@@ -23,7 +23,6 @@ import logging
 import re
 import threading
 import time
-from collections.abc import Mapping, Sequence
 from itertools import groupby
 from typing import TYPE_CHECKING, Any, Optional
 
@@ -31,11 +30,10 @@ from requests.exceptions import RequestException
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
-from rucio.common.config import config_get, config_get_bool
+from rucio.common.config import config_get, config_get_bool, config_get_float
 from rucio.common.exception import DatabaseException, TransferToolTimeout, TransferToolWrongAnswer
 from rucio.common.logging import setup_logging
 from rucio.common.stopwatch import Stopwatch
-from rucio.common.types import InternalAccount, LoggerFunction
 from rucio.common.utils import dict_chunks
 from rucio.core import request as request_core
 from rucio.core import transfer as transfer_core
@@ -44,12 +42,14 @@ from rucio.core.topology import ExpiringObjectCache, Topology
 from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
 from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, RequestState, RequestType
 from rucio.transfertool.fts3 import FTS3Transfertool
-from rucio.transfertool.transfertool import Transfertool
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
     from types import FrameType
 
+    from rucio.common.types import LoggerFunction
     from rucio.daemons.common import HeartbeatHandler
+    from rucio.transfertool.transfertool import Transfertool
 
 GRACEFUL_STOP = threading.Event()
 METRICS = MetricManager(module=__name__)
@@ -62,7 +62,7 @@ FILTER_TRANSFERTOOL = config_get('conveyor', 'filter_transfertool', False, None)
 def _fetch_requests(
         db_bulk: int,
         older_than: int,
-        activity_shares: Optional[Mapping[str, float]],
+        activity_shares: Optional['Mapping[str, float]'],
         transfertool: Optional[str],
         filter_transfertool: Optional[str],
         cached_topology: Optional[ExpiringObjectCache],
@@ -116,9 +116,9 @@ def _handle_requests(
         timeout: Optional[int],
         transfertool: str,
         transfer_stats_manager: request_core.TransferStatsManager,
-        oidc_account: Optional[str],
+        oidc_support: bool,
         *,
-        logger: LoggerFunction = logging.log,
+        logger: "LoggerFunction" = logging.log,
 ) -> None:
     transfs.sort(key=lambda t: (t['external_host'] or '',
                                 t['scope'].vo if multi_vo else '',
@@ -136,15 +136,9 @@ def _handle_requests(
 
                 transfertool_kwargs = {}
                 if transfertool_cls.external_name == FTS3Transfertool.external_name:
-                    account = None
-                    if oidc_account:
-                        if vo:
-                            account = InternalAccount(oidc_account, vo=vo)
-                        else:
-                            account = InternalAccount(oidc_account)
                     transfertool_kwargs.update({
                         'vo': vo,
-                        'oidc_account': account,
+                        'oidc_support': oidc_support,
                     })
 
                 transfertool_obj = transfertool_cls(external_host=external_host, **transfertool_kwargs)
@@ -161,12 +155,12 @@ def _handle_requests(
 
 def poller(
         once: bool = False,
-        activities: Optional[Sequence[str]] = None,
+        activities: Optional['Sequence[str]'] = None,
         sleep_time: int = 60,
         fts_bulk: int = 100,
         db_bulk: int = 1000,
         older_than: int = 60,
-        activity_shares: Optional[Mapping[str, float]] = None,
+        activity_shares: Optional['Mapping[str, float]'] = None,
         partition_wait_time: int = 10,
         transfertool: Optional[str] = TRANSFER_TOOL,
         filter_transfertool: Optional[str] = FILTER_TRANSFERTOOL,
@@ -176,13 +170,9 @@ def poller(
     """
     Main loop to check the status of a transfer primitive with a transfertool.
     """
-
-    timeout = config_get('conveyor', 'poll_timeout', default=None, raise_exception=False)
-    if timeout:
-        timeout = float(timeout)
-
+    timeout = config_get_float('conveyor', 'poll_timeout', default=None, raise_exception=False)
     multi_vo = config_get_bool('common', 'multi_vo', False, None)
-    oidc_account = config_get('conveyor', 'poller_oidc_account', False, None)
+    oidc_support = config_get_bool('conveyor', 'poller_oidc_support', default=False, raise_exception=False)
 
     executable = DAEMON_NAME
 
@@ -227,7 +217,7 @@ def poller(
             fts_bulk=fts_bulk,
             multi_vo=multi_vo,
             timeout=timeout,  # type: ignore (unclear if timeout is meant to be int or float)
-            oidc_account=oidc_account,
+            oidc_support=oidc_support,
             transfertool=transfertool,  # type: ignore (transfertool is not None)
             transfer_stats_manager=transfer_stats_manager,
         )
@@ -251,7 +241,7 @@ def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> N
 def run(
         once: bool = False,
         sleep_time: int = 60,
-        activities: Optional[Sequence[str]] = None,
+        activities: Optional['Sequence[str]'] = None,
         fts_bulk: int = 100,
         db_bulk: int = 1000,
         older_than: int = 60,
@@ -301,8 +291,8 @@ def run(
 
 
 def poll_transfers(
-        transfertool_obj: Transfertool,
-        transfers_by_eid: Mapping[str, Mapping[str, Any]],
+        transfertool_obj: 'Transfertool',
+        transfers_by_eid: 'Mapping[str, Mapping[str, Any]]',
         transfer_stats_manager: request_core.TransferStatsManager,
         timeout: "Optional[int]" = None,
         logger: "LoggerFunction" = logging.log
@@ -328,8 +318,8 @@ def poll_transfers(
 
 
 def _poll_transfers(
-        transfertool_obj: Transfertool,
-        transfers_by_eid: Mapping[str, Mapping[str, Any]],
+        transfertool_obj: 'Transfertool',
+        transfers_by_eid: 'Mapping[str, Mapping[str, Any]]',
         transfer_stats_manager: request_core.TransferStatsManager,
         timeout: "Optional[int]" = None,
         logger: "LoggerFunction" = logging.log