rucio 37.0.0rc3__py3-none-any.whl → 37.0.0rc4__py3-none-any.whl

rucio/daemons/cache/consumer.py

@@ -21,12 +21,13 @@ import logging
 import threading
 import time
 from traceback import format_exc
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import rucio.db.sqla.util
 from rucio.common import exception
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
 from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.stomp_utils import ListenerBase, StompConnectionManager
+from rucio.common.stomp_utils import StompConnectionManager
 from rucio.common.types import InternalScope, LoggerFunction
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import get_rse_id
@@ -35,6 +36,7 @@ from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_r
 if TYPE_CHECKING:
     from types import FrameType
 
+    from stomp import Connection
     from stomp.utils import Frame
 
 logging.getLogger("stomp").setLevel(logging.CRITICAL)
@@ -44,11 +46,35 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'cache-consumer'
 
 
-class AMQConsumer(ListenerBase):
+class AMQConsumer:
     """
     class Consumer
     """
 
+    def __init__(
+            self,
+            broker: str,
+            conn: "Connection",
+            logger: "LoggerFunction"
+    ):
+        """
+        __init__
+        """
+        self.__broker = broker
+        self.__conn = conn
+        self.__logger = logger
+
+    @METRICS.count_it
+    def on_heartbeat_timeout(self) -> None:
+        self.__conn.disconnect()
+
+    @METRICS.count_it
+    def on_error(self, frame: "Frame") -> None:
+        """
+        on_error
+        """
+        self.__logger(logging.ERROR, 'Message receive error: [%s] %s' % (self.__broker, frame.body))
+
     @METRICS.count_it
     def on_message(self, frame: "Frame") -> None:
         """
@@ -56,7 +82,7 @@ class AMQConsumer(ListenerBase):
         """
         try:
             msg = json.loads(frame.body)  # type: ignore
-            self._logger(logging.DEBUG, 'Message received: %s', msg)
+            self.__logger(logging.DEBUG, 'Message received: %s ' % msg)
             if isinstance(msg, dict) and 'operation' in msg.keys():
                 for f in msg['files']:
                     f['scope'] = InternalScope(f['scope'])
@@ -67,42 +93,81 @@ class AMQConsumer(ListenerBase):
 
                 rse_vo_str = msg['rse']
                 if 'vo' in msg and msg['vo'] != 'def':
-                    rse_vo_str = f"{rse_vo_str} on {msg['vo']}"
+                    rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo'])
                 if msg['operation'] == 'add_replicas':
-                    self._logger(logging.INFO, "add_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
+                    self.__logger(logging.INFO, 'add_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
                     add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
                 elif msg['operation'] == 'delete_replicas':
-                    self._logger(logging.INFO, "delete_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
+                    self.__logger(logging.INFO, 'delete_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
                     delete_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
             else:
-                self._logger(logging.DEBUG, 'Check failed: %s %s', isinstance(msg, dict), "operation" in msg.keys())
+                self.__logger(logging.DEBUG, 'Check failed: %s %s '
+                              % (isinstance(msg, dict), 'operation' in msg.keys()))
         except:
-            self._logger(logging.ERROR, str(format_exc()))
+            self.__logger(logging.ERROR, str(format_exc()))
 
 
-def consumer(id_: int, num_thread: int = 1, logger: LoggerFunction = logging.log) -> None:
+def consumer(id_: int, num_thread: int = 1) -> None:
     """
     Main loop to consume messages from the Rucio Cache producer.
     """
+
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
+
     logger(logging.INFO, 'Rucio Cache consumer starting')
 
-    conn_mgr = StompConnectionManager(config_section='messaging-cache', logger=logger)
+    brokers = config_get_list('messaging-cache', 'brokers')
+
+    use_ssl = config_get_bool('messaging-cache', 'use_ssl', default=True, raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-cache', 'username')
+        password = config_get('messaging-cache', 'password')
+    destination = config_get('messaging-cache', 'destination')
+    subscription_id = 'rucio-cache-messaging'
+
+    vhost = config_get('messaging-cache', 'broker_virtual_host', raise_exception=False)
+    port = config_get_int('messaging-cache', 'port')
+    reconnect_attempts = config_get_int('messaging-cache', 'reconnect_attempts', default=100)
+    ssl_key_file = config_get('messaging-cache', 'ssl_key_file', raise_exception=False)
+    ssl_cert_file = config_get('messaging-cache', 'ssl_cert_file', raise_exception=False)
+
+    stomp_conn_mngr = StompConnectionManager()
+    conns, _ = stomp_conn_mngr.re_configure(
+        brokers=brokers,
+        port=port,
+        use_ssl=use_ssl,
+        vhost=vhost,
+        reconnect_attempts=reconnect_attempts,
+        ssl_key_file=ssl_key_file,
+        ssl_cert_file=ssl_cert_file,
+        timeout=None,
+        logger=logger
+    )
 
     logger(logging.INFO, 'consumer started')
 
-    conn_mgr.set_listener_factory('rucio-cache-consumer', AMQConsumer, heartbeats=conn_mgr.config.heartbeats)
-
     while not GRACEFUL_STOP.is_set():
+        for conn in conns:
+            if not conn.is_connected():
+                host_port = conn.transport._Transport__host_and_ports[0]
+
+                logger(logging.INFO, 'connecting to %s' % host_port[0])
+                METRICS.counter('reconnect.{host}').labels(host=host_port[0]).inc()
+                conn.set_listener('rucio-cache-consumer', AMQConsumer(broker=host_port, conn=conn, logger=logger))
+                if not use_ssl:
+                    conn.connect(username, password)
+                else:
+                    conn.connect()
 
-        conn_mgr.subscribe(id_='rucio-cache-messaging', ack='auto')
+                conn.subscribe(destination=destination, ack='auto', id=subscription_id)
         time.sleep(1)
 
     logger(logging.INFO, 'graceful stop requested')
-    conn_mgr.disconnect()
+    stomp_conn_mngr.disconnect()
     logger(logging.INFO, 'graceful stop done')
 
 
-def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -115,19 +180,18 @@ def run(num_thread: int = 1) -> None:
     Starts up the rucio cache consumer thread
     """
     setup_logging(process_name=DAEMON_NAME)
-    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
 
     if rucio.db.sqla.util.is_old_db():
         raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
-    logger(logging.INFO, 'starting consumer thread')
-    threads = []
-    for i in range(num_thread):
-        con_thread = threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread, 'logger': logger})
-        con_thread.start()
-        threads.append(con_thread)
+    logging.info('starting consumer thread')
+    threads = [threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread})
+               for i in range(0, num_thread)]
+
+    [t.start() for t in threads]
 
-    logger(logging.INFO, 'waiting for interrupts')
+    logging.info('waiting for interrupts')
 
-    while [thread.join(timeout=3.14) for thread in threads if thread.is_alive()]:
-        pass
+    # Interruptible joins require a timeout.
+    while threads[0].is_alive():
+        [t.join(timeout=3.14) for t in threads]

rucio/daemons/conveyor/receiver.py

@@ -15,18 +15,22 @@
 """
 Conveyor is a daemon to manage file transfers.
 """
+
 import json
 import logging
+import socket
 import threading
 import time
 import traceback
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
+
+import stomp
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import setup_logging
 from rucio.common.policy import get_policy
-from rucio.common.stomp_utils import Connection, ListenerBase, StompConnectionManager
 from rucio.core import request as request_core
 from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MetricManager
@@ -49,22 +53,26 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'conveyor-receiver'
 
 
-class Receiver(ListenerBase):
+class Receiver:
 
-    def __init__(self,
-                 conn: Connection,
-                 id_: str,
-                 total_threads: int,
-                 transfer_stats_manager: request_core.TransferStatsManager,
-                 all_vos: bool = False,
-                 logger: "LoggerFunction" = logging.log,
-                 **kwargs: dict) -> None:
-        super().__init__(conn, logger, **kwargs)
+    def __init__(
+            self,
+            broker: str,
+            id_: str,
+            total_threads: int,
+            transfer_stats_manager: request_core.TransferStatsManager,
+            all_vos: bool = False
+    ):
         self.__all_vos = all_vos
+        self.__broker = broker
         self.__id = id_
         self.__total_threads = total_threads
         self._transfer_stats_manager = transfer_stats_manager
 
+    @METRICS.count_it
+    def on_error(self, frame: "Frame") -> None:
+        logging.error('[%s] %s' % (self.__broker, frame.body))
+
     @METRICS.count_it
     def on_message(self, frame: "Frame") -> None:
         msg = json.loads(frame.body)  # type: ignore
@@ -78,8 +86,7 @@ class Receiver(ListenerBase):
            and 'issuer' in msg['job_metadata'].keys() \
            and str(msg['job_metadata']['issuer']) == 'rucio':
 
-            if ('job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE'
-                                               or msg.get('job_multihop', False) is True)):
+            if 'job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE' or msg.get('job_multihop', False) is True):
                 METRICS.counter('message_rucio').inc()
 
                 self._perform_request_update(msg)
@@ -89,15 +96,13 @@ class Receiver(ListenerBase):
         self,
         msg: dict[str, Any],
         *,
-        session: "Session | None",
+        session: Optional["Session"] = None,
         logger: "LoggerFunction" = logging.log
     ) -> None:
         external_host = msg.get('endpnt', None)
         request_id = msg['file_metadata'].get('request_id', None)
         try:
-            tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host,
-                                                                         request_id=request_id,
-                                                                         fts_message=msg)
+            tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host, request_id=request_id, fts_message=msg)
             if tt_status_report.get_db_fields_to_update(session=session, logger=logger):  # type: ignore
                 logging.info('RECEIVED %s', tt_status_report)
 
@@ -115,39 +120,103 @@ class Receiver(ListenerBase):
             logging.critical(traceback.format_exc())
 
 
-def receiver(id_: str,
-             total_threads: int = 1,
-             all_vos: bool = False,
-             logger: "LoggerFunction" = logging.log):
+def receiver(
+        id_: str,
+        total_threads: int = 1,
+        all_vos: bool = False
+) -> None:
     """
     Main loop to consume messages from the FTS3 producer.
     """
-    logger(logging.INFO, 'receiver starting')
-
-    conn_mgr = StompConnectionManager(config_section='messaging-fts3', logger=logger)
 
-    logger(logging.INFO, 'receiver started')
-
-    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30),
+    logging.info('receiver starting')
+
+    brokers_alias = []
+    brokers_resolved = []
+    try:
+        brokers_alias = config_get_list('messaging-fts3', 'brokers')
+    except Exception:
+        raise Exception('Could not load brokers from configuration')
+
+    logging.info('resolving broker dns alias: %s' % brokers_alias)
+
+    brokers_resolved = []
+    for broker in brokers_alias:
+        addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        brokers_resolved.extend(ai[4][0] for ai in addrinfos)
+
+    logging.info('brokers resolved to %s', brokers_resolved)
+
+    logging.info('checking authentication method')
+    use_ssl = True
+    try:
+        use_ssl = config_get_bool('messaging-fts3', 'use_ssl')
+    except:
+        logging.info('could not find use_ssl in configuration -- please update your rucio.cfg')
+
+    port = config_get_int('messaging-fts3', 'port')
+    vhost = config_get('messaging-fts3', 'broker_virtual_host', raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-fts3', 'username')
+        password = config_get('messaging-fts3', 'password')
+        port = config_get_int('messaging-fts3', 'nonssl_port')
+
+    conns = []
+    for broker in brokers_resolved:
+        if not use_ssl:
+            logging.info('setting up username/password authentication: %s' % broker)
+        else:
+            logging.info('setting up ssl cert/key authentication: %s' % broker)
+        con = stomp.Connection12(host_and_ports=[(broker, port)],
+                                 vhost=vhost,
+                                 reconnect_attempts_max=999)
+        if use_ssl:
+            con.set_ssl(
+                key_file=config_get('messaging-fts3', 'ssl_key_file'),
+                cert_file=config_get('messaging-fts3', 'ssl_cert_file'),
+            )
+        conns.append(con)
+
+    logging.info('receiver started')
+
+    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30) as heartbeat_handler,
           request_core.TransferStatsManager() as transfer_stats_manager):
-
-        conn_mgr.set_listener_factory('rucio-messaging-fts3', Receiver,
-                                      id_=id_,
-                                      total_threads=total_threads,
-                                      transfer_stats_manager=transfer_stats_manager,
-                                      all_vos=all_vos,
-                                      heartbeats=conn_mgr.config.heartbeats)
-
         while not GRACEFUL_STOP.is_set():
 
-            conn_mgr.subscribe(id_='rucio-messaging-fts3', ack='auto')
-
+            _, _, logger = heartbeat_handler.live()
+
+            for conn in conns:
+
+                if not conn.is_connected():
+                    logger(logging.INFO, 'connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
+                    METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0].split('.')[0]).inc()
+
+                    conn.set_listener(
+                        'rucio-messaging-fts3',
+                        Receiver(
+                            broker=conn.transport._Transport__host_and_ports[0],
+                            id_=id_,
+                            total_threads=total_threads,
+                            transfer_stats_manager=transfer_stats_manager,
+                            all_vos=all_vos
+                        ))
+                    if not use_ssl:
+                        conn.connect(username, password, wait=True)
+                    else:
+                        conn.connect(wait=True)
+                    conn.subscribe(destination=config_get('messaging-fts3', 'destination'),
+                                   id='rucio-messaging-fts3',
+                                   ack='auto')
             time.sleep(1)
 
-        conn_mgr.disconnect()
+        for conn in conns:
+            try:
+                conn.disconnect()
+            except Exception:
+                pass
 
 
-def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -155,25 +224,26 @@ def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
     GRACEFUL_STOP.set()
 
 
-def run(once: bool = False, total_threads: int = 1) -> None:
+def run(
+        once: bool = False,
+        total_threads: int = 1
+) -> None:
     """
     Starts up the receiver thread
     """
     setup_logging(process_name=DAEMON_NAME)
-    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
 
     if rucio.db.sqla.util.is_old_db():
         raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
-    logger(logging.INFO, 'starting receiver thread')
-    threads = []
-    for i in range(total_threads):
-        rec_thread = threading.Thread(target=receiver,
-                                      kwargs={'id_': i, 'logger': logger, 'total_threads': total_threads})
-        rec_thread.start()
-        threads.append(rec_thread)
+    logging.info('starting receiver thread')
+    threads = [threading.Thread(target=receiver, kwargs={'id_': i,
+                                                         'total_threads': total_threads}) for i in range(0, total_threads)]
+
+    [thread.start() for thread in threads]
 
-    logger(logging.INFO, 'waiting for interrupts')
+    logging.info('waiting for interrupts')
 
-    while [thread.join(timeout=3.14) for thread in threads if thread.is_alive()]:
-        pass
+    # Interruptible joins require a timeout.
+    while threads:
+        threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.is_alive()]