rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/hermes/hermes.py

@@ -21,10 +21,8 @@ import datetime
 import functools
 import json
 import logging
-import random
 import re
 import smtplib
-import socket
 import sys
 import threading
 import time
@@ -33,28 +31,21 @@ from email.mime.text import MIMEText
 from typing import TYPE_CHECKING, Any, Optional, Union
 
 import requests
-import stomp
 from requests.auth import HTTPBasicAuth
 
 import rucio.db.sqla.util
-from rucio.common.config import (
-    config_get,
-    config_get_bool,
-    config_get_int,
-    config_get_list,
-)
+from rucio.common.config import config_get, config_get_bool, config_get_list
 from rucio.common.exception import DatabaseException
-from rucio.common.logging import setup_logging
+from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.stomp_utils import ListenerBase, StompConnectionManager
 from rucio.core.message import delete_messages, retrieve_messages
 from rucio.core.monitor import MetricManager
 from rucio.daemons.common import run_daemon
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Sequence
+    from collections.abc import Iterable
     from types import FrameType
 
-    from stomp.utils import Frame
-
     from rucio.common.types import LoggerFunction
     from rucio.daemons.common import HeartbeatHandler
 
@@ -76,267 +67,13 @@ def default(datetype: Union[datetime.date, datetime.datetime]) -> str:
         return datetype.isoformat()
 
 
-class HermesListener(stomp.ConnectionListener):
+class HermesListener(ListenerBase):
     """
     Hermes Listener
     """
 
-    def __init__(self, broker: str):
-        """
-        __init__
-        """
-        self.__broker = broker
-
-    def on_error(self, frame: "Frame") -> None:
-        """
-        Error handler
-        """
-        logging.error("[broker] [%s]: %s", self.__broker, frame.body)
-
-
-def setup_activemq(
-        logger: "LoggerFunction"
-) -> tuple[
-    Optional[list[stomp.Connection12]],
-    Optional[str],
-    Optional[str],
-    Optional[str],
-    Optional[bool]
-]:
-    """
-    Deliver messages to ActiveMQ
 
-    :param logger:             The logger object.
-    """
-
-    logger(logging.INFO, "[broker] Resolving brokers")
-
-    brokers_alias = []
-    brokers_resolved = []
-    try:
-        brokers_alias = [
-            broker.strip()
-            for broker in config_get("messaging-hermes", "brokers").split(",")
-        ]
-    except:
-        raise Exception("Could not load brokers from configuration")
-
-    logger(logging.INFO, "[broker] Resolving broker dns alias: %s", brokers_alias)
-    brokers_resolved = []
-    for broker in brokers_alias:
-        try:
-            addrinfos = socket.getaddrinfo(
-                broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP
-            )
-            brokers_resolved.extend(ai[4][0] for ai in addrinfos)
-        except socket.gaierror as ex:
-            logger(
-                logging.ERROR,
-                "[broker] Cannot resolve domain name %s (%s)",
-                broker,
-                str(ex),
-            )
-
-    logger(logging.DEBUG, "[broker] Brokers resolved to %s", brokers_resolved)
-
-    if not brokers_resolved:
-        logger(logging.FATAL, "[broker] No brokers resolved.")
-        return None, None, None, None, None
-
-    broker_timeout = 3
-    if not broker_timeout:  # Allow zero in config
-        broker_timeout = None
-
-    logger(logging.INFO, "[broker] Checking authentication method")
-    use_ssl = True
-    try:
-        use_ssl = config_get_bool("messaging-hermes", "use_ssl")
-    except:
-        logger(
-            logging.INFO,
-            "[broker] Could not find use_ssl in configuration -- please update your rucio.cfg",
-        )
-
-    port = config_get_int("messaging-hermes", "port")
-    vhost = config_get("messaging-hermes", "broker_virtual_host", raise_exception=False)
-    username = None
-    password = None
-    if not use_ssl:
-        username = config_get("messaging-hermes", "username")
-        password = config_get("messaging-hermes", "password")
-        port = config_get_int("messaging-hermes", "nonssl_port")
-
-    conns = []
-    for broker in brokers_resolved:
-        if not use_ssl:
-            logger(
-                logging.INFO,
-                "[broker] setting up username/password authentication: %s",
-                broker,
-            )
-        else:
-            logger(
-                logging.INFO,
-                "[broker] setting up ssl cert/key authentication: %s",
-                broker,
-            )
-
-        con = stomp.Connection12(
-            host_and_ports=[(broker, port)],
-            vhost=vhost,
-            keepalive=True,
-            timeout=broker_timeout,
-        )
-        if use_ssl:
-            con.set_ssl(
-                key_file=config_get("messaging-hermes", "ssl_key_file"),
-                cert_file=config_get("messaging-hermes", "ssl_cert_file"),
-            )
-
-        con.set_listener(
-            "rucio-hermes", HermesListener(con.transport._Transport__host_and_ports[0])
-        )
-
-        conns.append(con)
-    destination = config_get("messaging-hermes", "destination")
-    return conns, destination, username, password, use_ssl
-
-
-def deliver_to_activemq(
-    messages: "Iterable[dict[str, Any]]",
-    conns: "Sequence[stomp.Connection12]",
-    destination: str,
-    username: str,
-    password: str,
-    use_ssl: bool,
-    logger: "LoggerFunction"
-) -> list[str]:
-    """
-    Deliver messages to ActiveMQ
-
-    :param messages:           The list of messages.
-    :param conns:              A list of connections.
-    :param destination:        The destination topic or queue.
-    :param username:           The username if no SSL connection.
-    :param password:           The username if no SSL connection.
-    :param use_ssl:            Boolean to choose if SSL connection is used.
-    :param logger:             The logger object.
-
-    :returns:                  List of message_id to delete
-    """
-    to_delete = []
-    for message in messages:
-        try:
-            conn = random.sample(conns, 1)[0]
-            if not conn.is_connected():
-                host_and_ports = conn.transport._Transport__host_and_ports[0][0]
-                RECONNECT_COUNTER.labels(host=host_and_ports.split(".")[0]).inc()
-                if not use_ssl:
-                    logger(
-                        logging.INFO,
-                        "[broker] - connecting with USERPASS to %s",
-                        host_and_ports,
-                    )
-                    conn.connect(username, password, wait=True)
-                else:
-                    logger(
-                        logging.INFO,
-                        "[broker] - connecting with SSL to %s",
-                        host_and_ports,
-                    )
-                    conn.connect(wait=True)
-
-            conn.send(
-                body=json.dumps(
-                    {
-                        "event_type": str(message["event_type"]).lower(),
-                        "payload": message["payload"],
-                        "created_at": str(message["created_at"]),
-                    }
-                ),
-                destination=destination,
-                headers={
-                    "persistent": "true",
-                    "event_type": str(message["event_type"]).lower(),
-                },
-            )
-
-            to_delete.append(message["id"])
-        except ValueError:
-            logger(
-                logging.ERROR,
-                "[broker] Cannot serialize payload to JSON: %s",
-                str(message["payload"]),
-            )
-            to_delete.append(message["id"])
-            continue
-        except stomp.exception.NotConnectedException as error:
-            logger(
-                logging.WARNING,
-                "[broker] Could not deliver message due to NotConnectedException: %s",
-                str(error),
-            )
-            continue
-        except stomp.exception.ConnectFailedException as error:
-            logger(
-                logging.WARNING,
-                "[broker] Could not deliver message due to ConnectFailedException: %s",
-                str(error),
-            )
-            continue
-        except Exception as error:
-            logger(logging.ERROR, "[broker] Could not deliver message: %s", str(error))
-            continue
-
-        if str(message["event_type"]).lower().startswith("transfer") or str(
-            message["event_type"]
-        ).lower().startswith("stagein"):
-            logger(
-                logging.DEBUG,
-                "[broker] - event_type: %s, scope: %s, name: %s, rse: %s, request-id: %s, transfer-id: %s, created_at: %s",
-                str(message["event_type"]).lower(),
-                message["payload"].get("scope", None),
-                message["payload"].get("name", None),
-                message["payload"].get("dst-rse", None),
-                message["payload"].get("request-id", None),
-                message["payload"].get("transfer-id", None),
-                str(message["created_at"]),
-            )
-
-        elif str(message["event_type"]).lower().startswith("dataset"):
-            logger(
-                logging.DEBUG,
-                "[broker] - event_type: %s, scope: %s, name: %s, rse: %s, rule-id: %s, created_at: %s)",
-                str(message["event_type"]).lower(),
-                message["payload"].get("scope", None),
-                message["payload"].get("name", None),
-                message["payload"].get("rse", None),
-                message["payload"].get("rule_id", None),
-                str(message["created_at"]),
-            )
-
-        elif str(message["event_type"]).lower().startswith("deletion"):
-            if "url" not in message["payload"]:
-                message["payload"]["url"] = "unknown"
-            logger(
-                logging.DEBUG,
-                "[broker] - event_type: %s, scope: %s, name: %s, rse: %s, url: %s, created_at: %s)",
-                str(message["event_type"]).lower(),
-                message["payload"].get("scope", None),
-                message["payload"].get("name", None),
-                message["payload"].get("rse", None),
-                message["payload"].get("url", None),
-                str(message["created_at"]),
-            )
-        else:
-            logger(logging.DEBUG, "[broker] Other message: %s", message)
-    return to_delete
-
-
-def deliver_emails(
-        messages: "Iterable[dict[str, Any]]",
-        logger: "LoggerFunction"
-) -> list[str]:
+def deliver_emails(messages: "Iterable[dict[str, Any]]", logger: "LoggerFunction") -> list[int]:
     """
     Sends emails
 
@@ -400,15 +137,16 @@ def submit_to_elastic(
 
     for message in messages:
         text += '{ "index":{ } }\n%s\n' % json.dumps(message, default=default)
-    res = requests.post(
-        endpoint, data=text, headers={"Content-Type": "application/json"}, auth=auth
-    )
+    res = requests.post(endpoint,
+                        data=text,
+                        headers={"Content-Type": "application/json"},
+                        auth=auth)
     return res.status_code
 
 
 def aggregate_to_influx(
     messages: "Iterable[dict[str, Any]]",
-    bin_size: int,
+    bin_size: str,
     endpoint: str,
     logger: "LoggerFunction"
 ) -> int:
@@ -432,11 +170,9 @@ def aggregate_to_influx(
         payload = message["payload"]
         if event_type in ["transfer-failed", "transfer-done"]:
             if not payload["transferred_at"]:
-                logger(
-                    logging.WARNING,
-                    "No transferred_at for message. Reason : %s",
-                    payload["reason"],
-                )
+                logger(logging.WARNING,
+                       "No transferred_at for message. Reason : %s",
+                       payload["reason"])
                 continue
             transferred_at = time.strptime(
                 payload["transferred_at"], "%Y-%m-%d %H:%M:%S"
@@ -452,11 +188,7 @@ def aggregate_to_influx(
                 payload["activity"],
             )
             activity = re.sub(" ", r"\ ", activity)
-            key = "transfer,activity=%s,src_rse=%s,dst_rse=%s" % (
-                activity,
-                src_rse,
-                dest_rse,
-            )
+            key = f"transfer,activity={activity!s},src_rse={src_rse!s},dst_rse={dest_rse!s}"
             if key not in bins[transferred_at]:
                 bins[transferred_at][key] = [0, 0, 0, 0]
             if event_type == "transfer-done":
@@ -476,7 +208,7 @@ def aggregate_to_influx(
             if created_at not in bins:
                 bins[created_at] = {}
             rse = payload["rse"]
-            key = "deletion,rse=%s" % (rse)
+            key = f"deletion,rse={rse!s}"
             if key not in bins[created_at]:
                 bins[created_at][key] = [0, 0, 0, 0]
             if event_type == "deletion-done":
@@ -485,31 +217,22 @@ def aggregate_to_influx(
             if event_type == "deletion-failed":
                 bins[created_at][key][2] += 1
                 bins[created_at][key][3] += payload["bytes"]
+
     points = ""
-    for timestamp in bins:
-        for entry in bins[timestamp]:
-            metrics = bins[timestamp][entry]
-            event_type = entry.split(",")[0]
-            point = (
-                "%s nb_%s_done=%s,bytes_%s_done=%s,nb_%s_failed=%s,bytes_%s_failed=%s %s"
-                % (
-                    entry,
-                    event_type,
-                    metrics[0],
-                    event_type,
-                    metrics[1],
-                    event_type,
-                    metrics[2],
-                    event_type,
-                    metrics[3],
-                    timestamp,
-                )
-            )
-            points += point
-            points += "\n"
+    for timestamp, entries in bins.items():
+        for key, metrics in entries.items():
+            event_type = key.split(",")[0]
+            points += (f"{key!s} "
+                       f"nb_{event_type!s}_done={metrics[0]!s},"
+                       f"bytes_{event_type!s}_done={metrics[1]!s},"
+                       f"nb_{event_type!s}_failed={metrics[2]!s},"
+                       f"bytes_{event_type!s}_failed={metrics[3]!s} "
+                       rf"{timestamp!s}\n")
+
     influx_token = config_get("hermes", "influxdb_token", False, None)
+    headers = {}
     if influx_token:
-        headers = {"Authorization": "Token %s" % influx_token}
+        headers["Authorization"] = f"Token {influx_token!s}"
     if points:
         res = requests.post(endpoint, headers=headers, data=points)
         logger(logging.DEBUG, "%s", str(res.text))
@@ -517,6 +240,69 @@ def aggregate_to_influx(
     return 204
 
 
+def build_message_dict(
+        bulk: int,
+        thread: int,
+        total_threads: int,
+        message_dict: dict[str, list[dict[str, Any]]],
+        logger: "LoggerFunction",
+        service: Optional[str] = None,
+) -> None:
+    """
+    Retrieves messages from the database and builds a dictionary with the keys being the services, and the values a list of the messages (built up of dictionary / json information)
+
+    :param bulk:               Integer for number of messages to retrieve.
+    :param thread:             Passed to thread in retrieve_messages for Identifier of the caller thread as an integer.
+    :param total_threads:      Passed to total_threads for Maximum number of threads as an integer.
+    :param message_dict:       Either empty dictionary to be built, or build upon when using query_by_service.
+    :param logger:             The logger object.
+    :param service:            When passed, only returns messages table for this specific service.
+
+    :returns:                  None, but builds on the dictionary message_dict passed to this fuction (for when querying multiple services).
+    """
+    start_time = time.time()
+    messages = retrieve_messages(
+        bulk=bulk,
+        old_mode=False,
+        thread=thread,
+        total_threads=total_threads,
+        service_filter=service,
+    )
+
+    if messages:
+        if service is not None:
+            # query_by_service dictionary build behaviour
+            message_dict[service] = messages.copy()
+            logger(
+                logging.DEBUG,
+                "Retrieved %i messages retrieved in %s seconds for %s service.",
+                len(messages),
+                time.time() - start_time,
+                service,
+            )
+
+        else:
+            # default dictionary build behaviour
+            for message in messages:
+                service = message["services"]
+                if service is not None:
+                    if service not in message_dict:
+                        message_dict[service] = []
+                    message_dict[service].append(message)
+        logger(
+            logging.DEBUG,
+            "Retrieved %i messages retrieved in %s seconds",
+            len(messages),
+            time.time() - start_time,
+        )
+    else:
+        logger(
+            logging.INFO,
+            "No messages retrieved in %s seconds",
+            time.time() - start_time,
+        )
+
+
 def hermes(once: bool = False, bulk: int = 1000, sleep_time: int = 10) -> None:
     """
     Creates a Hermes Worker that can submit messages to different services (InfluXDB, ElasticSearch, ActiveMQ)
@@ -573,42 +359,36 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
             logger(logging.ERROR, str(err))
     conns = None
     if "activemq" in services_list:
-        try:
-            conns, destination, username, password, use_ssl = setup_activemq(logger)
-            if not conns:
-                logger(
-                    logging.ERROR,
-                    "ActiveMQ defined in the services list, cannot be setup",
-                )
-        except Exception as err:
-            logger(logging.ERROR, str(err))
+        conn_mgr = StompConnectionManager(config_section='messaging-hermes', logger=logger)
+        conn_mgr.set_listener_factory("rucio-hermes", HermesListener, heartbeats=conn_mgr.config.heartbeats)
 
     worker_number, total_workers, logger = heartbeat_handler.live()
     message_dict = {}
-    message_ids = []
-    start_time = time.time()
-    messages = retrieve_messages(
-        bulk=bulk,
-        old_mode=False,
-        thread=worker_number,
-        total_threads=total_workers,
-    )
-
-    to_delete = []
-    if messages:
-        for message in messages:
-            service = message["services"]
-            if service not in message_dict:
-                message_dict[service] = []
-            message_dict[service].append(message)
-            message_ids.append(message["id"])
-        logger(
-            logging.DEBUG,
-            "Retrieved %i messages retrieved in %s seconds",
-            len(messages),
-            time.time() - start_time,
+    query_by_service = config_get_bool("hermes", "query_by_service", default=False)
+
+    # query_by_service is a toggleable behaviour switch between collecting bulk number of messages across all services when false, to collecting bulk messages from each service when true.
+    if query_by_service:
+        for service in services_list:
+            build_message_dict(
+                bulk=bulk,
+                thread=worker_number,
+                total_threads=total_workers,
+                message_dict=message_dict,
+                logger=logger,
+                service=service,
+            )
+    else:
+        build_message_dict(
+            bulk=bulk,
+            thread=worker_number,
+            total_threads=total_workers,
+            message_dict=message_dict,
+            logger=logger
         )
 
+    if message_dict:
+        to_delete = []
+
         if "influx" in message_dict and influx_endpoint:
             # For influxDB, bulk submission, either everything succeeds or fails
             t_time = time.time()
@@ -688,15 +468,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
         if "activemq" in message_dict and conns:
             t_time = time.time()
             try:
-                messages_sent = deliver_to_activemq(
-                    messages=message_dict["activemq"],
-                    conns=conns,
-                    destination=destination,  # type: ignore (argument could be None)
-                    username=username,  # type: ignore (argument could be None)
-                    password=password,  # type: ignore (argument could be None)
-                    use_ssl=use_ssl,  # type: ignore (argument could be None)
-                    logger=logger,
-                )
+                messages_sent = conn_mgr.deliver_messages(messages=message_dict["activemq"])
                 logger(
                     logging.INFO,
                     "%s messages successfully submitted to ActiveMQ in %s seconds",
@@ -717,6 +489,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
             "updated_at": message["created_at"],
             "payload": str(message["payload"]),
             "event_type": message["event_type"],
+            "services": message["services"]
         }
         for message in to_delete
     ]
@@ -744,31 +517,18 @@ def run(
     Starts up the hermes threads.
     """
     setup_logging(process_name=DAEMON_NAME)
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
 
     if rucio.db.sqla.util.is_old_db():
         raise DatabaseException("Database was not updated, daemon won't start")
 
-    logging.info("starting hermes threads")
-    thread_list = [
-        threading.Thread(
-            target=hermes,
-            kwargs={
-                "once": once,
-                "bulk": bulk,
-                "sleep_time": sleep_time,
-            },
-        )
-        for _ in range(0, threads)
-    ]
+    logger(logging.INFO, "starting hermes threads")
+    thread_list = []
+    for _ in range(threads):
+        her_thread = threading.Thread(target=hermes, kwargs={"once": once, "bulk": bulk, "sleep_time": sleep_time})
+        her_thread.start()
+        thread_list.append(her_thread)
 
-    for thrd in thread_list:
-        thrd.start()
-
-    logging.debug(thread_list)
-    # Interruptible joins require a timeout.
-    while thread_list:
-        thread_list = [
-            thread.join(timeout=3.14)
-            for thread in thread_list
-            if thread and thread.is_alive()
-        ]
+    logger(logging.DEBUG, thread_list)
+    while [thread.join(timeout=3.14) for thread in thread_list if thread.is_alive()]:
+        pass

rucio/daemons/judge/evaluator.py

@@ -91,9 +91,9 @@ def run_once(
     dids = get_updated_dids(total_workers=total_workers,
                             worker_number=worker_number,
                             limit=did_limit,
-                            blocked_dids=[(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])
+                            blocked_dids=[(InternalScope(key[0], from_external=False), key[1]) for key in paused_dids])
     logger(logging.DEBUG, 'index query time %f fetch size is %d (%d blocked)', time.time() - start, len(dids),
-           len([(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids]))
+           len([(InternalScope(key[0], from_external=False), key[1]) for key in paused_dids]))
 
     # If the list is empty, sent the worker to sleep
     if not dids:

rucio/daemons/oauthmanager/oauthmanager.py

@@ -53,7 +53,7 @@ graceful_stop = threading.Event()
 DAEMON_NAME = 'oauth-manager'
 
 
-def OAuthManager(once: bool = False, max_rows: int = 100, sleep_time: int = 300) -> None:
+def oauth_manager(once: bool = False, max_rows: int = 100, sleep_time: int = 300) -> None:
     """
     Main loop to delete all expired tokens, refresh tokens eligible
     for refresh and delete all expired OAuth session parameters.
@@ -178,10 +178,10 @@ def run(once: bool = False, threads: int = 1, max_rows: int = 100, sleep_time: i
         raise DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
-        OAuthManager(once, max_rows, sleep_time)
+        oauth_manager(once, max_rows, sleep_time)
     else:
         logging.info('OAuth Manager starting %s threads', str(threads))
-        threads = [threading.Thread(target=OAuthManager,
+        threads = [threading.Thread(target=oauth_manager,
                                     kwargs={'once': once,
                                             'max_rows': max_rows,
                                             'sleep_time': sleep_time}) for i in range(0, threads)]

rucio/daemons/reaper/dark_reaper.py

@@ -44,6 +44,7 @@ if TYPE_CHECKING:
     from types import FrameType
     from typing import Optional
 
+    from rucio.common.types import LFNDict
     from rucio.daemons.common import HeartbeatHandler
 
 logging.getLogger("requests").setLevel(logging.CRITICAL)
@@ -120,10 +121,13 @@ def run_once(
                 if replica['scope']:
                     scope = replica['scope'].external
                 try:
+                    lfn: "LFNDict" = {
+                        'scope': scope,
+                        'name': replica['name'],
+                        'path': replica['path']
+                    }
                     pfn = str(list(rsemgr.lfns2pfns(rse_settings=rse_info,
-                                                    lfns=[{'scope': scope,
-                                                           'name': replica['name'],
-                                                           'path': replica['path']}],
+                                                    lfns=[lfn],
                                                     operation='delete',
                                                     scheme=scheme).values())[0])
                     logger(logging.INFO, 'Deletion ATTEMPT of %s:%s as %s on %s', scope, replica['name'], pfn, rse)