rucio 37.0.0rc3__py3-none-any.whl → 37.1.0__py3-none-any.whl

rucio/daemons/conveyor/receiver.py

@@ -15,18 +15,22 @@
 """
 Conveyor is a daemon to manage file transfers.
 """
+
 import json
 import logging
+import socket
 import threading
 import time
 import traceback
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
+
+import stomp
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import setup_logging
 from rucio.common.policy import get_policy
-from rucio.common.stomp_utils import Connection, ListenerBase, StompConnectionManager
 from rucio.core import request as request_core
 from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MetricManager
@@ -49,22 +53,26 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'conveyor-receiver'
 
 
-class Receiver(ListenerBase):
+class Receiver:
 
-    def __init__(self,
-                 conn: Connection,
-                 id_: str,
-                 total_threads: int,
-                 transfer_stats_manager: request_core.TransferStatsManager,
-                 all_vos: bool = False,
-                 logger: "LoggerFunction" = logging.log,
-                 **kwargs: dict) -> None:
-        super().__init__(conn, logger, **kwargs)
+    def __init__(
+            self,
+            broker: str,
+            id_: str,
+            total_threads: int,
+            transfer_stats_manager: request_core.TransferStatsManager,
+            all_vos: bool = False
+    ):
         self.__all_vos = all_vos
+        self.__broker = broker
         self.__id = id_
         self.__total_threads = total_threads
         self._transfer_stats_manager = transfer_stats_manager
 
+    @METRICS.count_it
+    def on_error(self, frame: "Frame") -> None:
+        logging.error('[%s] %s' % (self.__broker, frame.body))
+
     @METRICS.count_it
     def on_message(self, frame: "Frame") -> None:
         msg = json.loads(frame.body)  # type: ignore
@@ -78,8 +86,7 @@ class Receiver(ListenerBase):
            and 'issuer' in msg['job_metadata'].keys() \
            and str(msg['job_metadata']['issuer']) == 'rucio':
 
-            if ('job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE'
-                                               or msg.get('job_multihop', False) is True)):
+            if 'job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE' or msg.get('job_multihop', False) is True):
                 METRICS.counter('message_rucio').inc()
 
                 self._perform_request_update(msg)
@@ -89,15 +96,13 @@ class Receiver(ListenerBase):
         self,
         msg: dict[str, Any],
         *,
-        session: "Session | None",
+        session: Optional["Session"] = None,
         logger: "LoggerFunction" = logging.log
     ) -> None:
         external_host = msg.get('endpnt', None)
         request_id = msg['file_metadata'].get('request_id', None)
         try:
-            tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host,
-                                                                         request_id=request_id,
-                                                                         fts_message=msg)
+            tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host, request_id=request_id, fts_message=msg)
             if tt_status_report.get_db_fields_to_update(session=session, logger=logger):  # type: ignore
                 logging.info('RECEIVED %s', tt_status_report)
 
@@ -115,39 +120,103 @@ class Receiver(ListenerBase):
             logging.critical(traceback.format_exc())
 
 
-def receiver(id_: str,
-             total_threads: int = 1,
-             all_vos: bool = False,
-             logger: "LoggerFunction" = logging.log):
+def receiver(
+        id_: str,
+        total_threads: int = 1,
+        all_vos: bool = False
+) -> None:
     """
     Main loop to consume messages from the FTS3 producer.
     """
-    logger(logging.INFO, 'receiver starting')
-
-    conn_mgr = StompConnectionManager(config_section='messaging-fts3', logger=logger)
 
-    logger(logging.INFO, 'receiver started')
-
-    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30),
+    logging.info('receiver starting')
+
+    brokers_alias = []
+    brokers_resolved = []
+    try:
+        brokers_alias = config_get_list('messaging-fts3', 'brokers')
+    except Exception:
+        raise Exception('Could not load brokers from configuration')
+
+    logging.info('resolving broker dns alias: %s' % brokers_alias)
+
+    brokers_resolved = []
+    for broker in brokers_alias:
+        addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        brokers_resolved.extend(ai[4][0] for ai in addrinfos)
+
+    logging.info('brokers resolved to %s', brokers_resolved)
+
+    logging.info('checking authentication method')
+    use_ssl = True
+    try:
+        use_ssl = config_get_bool('messaging-fts3', 'use_ssl')
+    except:
+        logging.info('could not find use_ssl in configuration -- please update your rucio.cfg')
+
+    port = config_get_int('messaging-fts3', 'port')
+    vhost = config_get('messaging-fts3', 'broker_virtual_host', raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-fts3', 'username')
+        password = config_get('messaging-fts3', 'password')
+        port = config_get_int('messaging-fts3', 'nonssl_port')
+
+    conns = []
+    for broker in brokers_resolved:
+        if not use_ssl:
+            logging.info('setting up username/password authentication: %s' % broker)
+        else:
+            logging.info('setting up ssl cert/key authentication: %s' % broker)
+        con = stomp.Connection12(host_and_ports=[(broker, port)],
+                                 vhost=vhost,
+                                 reconnect_attempts_max=999)
+        if use_ssl:
+            con.set_ssl(
+                key_file=config_get('messaging-fts3', 'ssl_key_file'),
+                cert_file=config_get('messaging-fts3', 'ssl_cert_file'),
+            )
+        conns.append(con)
+
+    logging.info('receiver started')
+
+    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30) as heartbeat_handler,
           request_core.TransferStatsManager() as transfer_stats_manager):
-
-        conn_mgr.set_listener_factory('rucio-messaging-fts3', Receiver,
-                                      id_=id_,
-                                      total_threads=total_threads,
-                                      transfer_stats_manager=transfer_stats_manager,
-                                      all_vos=all_vos,
-                                      heartbeats=conn_mgr.config.heartbeats)
-
         while not GRACEFUL_STOP.is_set():
 
-            conn_mgr.subscribe(id_='rucio-messaging-fts3', ack='auto')
-
+            _, _, logger = heartbeat_handler.live()
+
+            for conn in conns:
+
+                if not conn.is_connected():
+                    logger(logging.INFO, 'connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
+                    METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0].split('.')[0]).inc()
+
+                    conn.set_listener(
+                        'rucio-messaging-fts3',
+                        Receiver(
+                            broker=conn.transport._Transport__host_and_ports[0],
+                            id_=id_,
+                            total_threads=total_threads,
+                            transfer_stats_manager=transfer_stats_manager,
+                            all_vos=all_vos
+                        ))
+                    if not use_ssl:
+                        conn.connect(username, password, wait=True)
+                    else:
+                        conn.connect(wait=True)
+                    conn.subscribe(destination=config_get('messaging-fts3', 'destination'),
+                                   id='rucio-messaging-fts3',
+                                   ack='auto')
             time.sleep(1)
 
-        conn_mgr.disconnect()
+        for conn in conns:
+            try:
+                conn.disconnect()
+            except Exception:
+                pass
 
 
-def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -155,25 +224,26 @@ def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
     GRACEFUL_STOP.set()
 
 
-def run(once: bool = False, total_threads: int = 1) -> None:
+def run(
+        once: bool = False,
+        total_threads: int = 1
+) -> None:
     """
     Starts up the receiver thread
     """
     setup_logging(process_name=DAEMON_NAME)
-    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
 
     if rucio.db.sqla.util.is_old_db():
         raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
-    logger(logging.INFO, 'starting receiver thread')
-    threads = []
-    for i in range(total_threads):
-        rec_thread = threading.Thread(target=receiver,
-                                      kwargs={'id_': i, 'logger': logger, 'total_threads': total_threads})
-        rec_thread.start()
-        threads.append(rec_thread)
+    logging.info('starting receiver thread')
+    threads = [threading.Thread(target=receiver, kwargs={'id_': i,
+                                                         'total_threads': total_threads}) for i in range(0, total_threads)]
+
+    [thread.start() for thread in threads]
 
-    logger(logging.INFO, 'waiting for interrupts')
+    logging.info('waiting for interrupts')
 
-    while [thread.join(timeout=3.14) for thread in threads if thread.is_alive()]:
-        pass
+    # Interruptible joins require a timeout.
+    while threads:
+        threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.is_alive()]