rucio 37.0.0rc2__tar.gz → 37.0.0rc4__tar.gz

{rucio-37.0.0rc2 → rucio-37.0.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rucio
-Version: 37.0.0rc2
+Version: 37.0.0rc4
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

rucio-37.0.0rc4/lib/rucio/common/stomp_utils.py

@@ -0,0 +1,159 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Common utility functions for stomp connections
+"""
+
+import logging
+import socket
+from time import monotonic
+from typing import TYPE_CHECKING
+
+from stomp import Connection
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+    from typing import Any
+
+    LoggerFunction = Callable[..., Any]
+
+
+def resolve_ips(fqdns: "Sequence[str]", logger: "LoggerFunction" = logging.log):
+    logger(logging.DEBUG, 'resolving dns aliases: %s' % fqdns)
+    resolved = []
+    for fqdn in fqdns:
+        addrinfos = socket.getaddrinfo(fqdn, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        resolved.extend(ai[4][0] for ai in addrinfos)
+    logger(logging.DEBUG, 'dns aliases resolved to %s', resolved)
+    return resolved
+
+
+class StompConnectionManager:
+
+    def __init__(self):
+        self._brokers = None
+        self._port = None
+        self._use_ssl = None
+        self._vhost = None
+        self._reconnect_attempts = None
+        self._ssl_key_file = None
+        self._timeout = None
+        self._heartbeats = None
+
+        self._connections = {}
+
+    def is_stalled(self, connection: Connection, *, logger: "LoggerFunction" = logging.log):
+        if not connection.is_connected():
+            return True
+
+        if self._heartbeats and getattr(connection, 'received_heartbeat') and connection.received_heartbeat:
+            heartbeat_period_seconds = max(0, self._heartbeats[0], self._heartbeats[1]) / 1000
+
+            if not heartbeat_period_seconds:
+                return False
+
+            now = monotonic()
+            if connection.received_heartbeat + 10 * heartbeat_period_seconds < now:
+                logger(logging.WARNING, "Stomp connection missed heartbeats for a long time")
+                return True
+
+        return False
+
+    def disconnect(self):
+        for conn in self._connections.values():
+            if not conn.is_connected():
+                conn.disconnect()
+
+    def re_configure(
+            self,
+            brokers: "Sequence[str]",
+            port: int,
+            use_ssl: bool,
+            vhost,
+            reconnect_attempts: int,
+            ssl_key_file,
+            ssl_cert_file,
+            timeout,
+            heartbeats=(0, 1000),
+            *,
+            logger: "LoggerFunction" = logging.log
+    ) -> tuple[list, list]:
+
+        configuration_changed = any([
+            self._brokers != brokers,
+            self._port != port,
+            self._use_ssl != use_ssl,
+            self._vhost != vhost,
+            self._reconnect_attempts != reconnect_attempts,
+            self._ssl_key_file != ssl_key_file,
+            self._timeout != timeout,
+            self._heartbeats != heartbeats,
+        ])
+        if configuration_changed:
+            self._brokers = brokers
+            self._port = port
+            self._use_ssl = use_ssl
+            self._vhost = vhost
+            self._reconnect_attempts = reconnect_attempts
+            self._ssl_key_file = ssl_key_file
+            self._timeout = timeout
+            self._heartbeats = heartbeats
+
+        current_remotes = set(self._connections)
+        desired_remotes = set((ip, port) for ip in resolve_ips(brokers, logger=logger))
+
+        if configuration_changed:
+            # Re-create all connections
+            to_delete = current_remotes
+            to_create = desired_remotes
+        else:
+            to_delete = current_remotes.difference(desired_remotes)
+            to_create = desired_remotes.difference(current_remotes)
+
+            for remote in current_remotes.intersection(desired_remotes):
+                conn = self._connections[remote]
+
+                if self.is_stalled(conn, logger=logger):
+                    # Re-create stalled connections
+                    to_delete.add(remote)
+                    to_create.add(remote)
+
+        deleted_conns = []
+        for remote in to_delete:
+            conn = self._connections.pop(remote)
+            if conn.is_connected():
+                conn.disconnect()
+            deleted_conns.append(to_delete)
+
+        created_conns = []
+        for remote in to_create:
+            conn = Connection(
+                host_and_ports=[remote],
+                vhost=vhost,
+                timeout=timeout,
+                heartbeats=heartbeats,
+                reconnect_attempts_max=reconnect_attempts
+            )
+            if use_ssl:
+                conn.set_ssl(key_file=ssl_key_file, cert_file=ssl_cert_file)
+            self._connections[remote] = conn
+            created_conns.append(conn)
+
+        if not to_delete and not to_create:
+            logger(logging.INFO, "Stomp connections didn't change")
+        else:
+            logger(logging.INFO, f"Stomp connections refreshed. Deleted: {list(to_delete)}. Added: {list(to_create)}")
+
+        return created_conns, deleted_conns

rucio-37.0.0rc4/lib/rucio/daemons/cache/consumer.py

@@ -0,0 +1,197 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Cache consumer is a daemon to retrieve rucio cache operation information to synchronize rucio catalog.
+"""
+
+import json
+import logging
+import threading
+import time
+from traceback import format_exc
+from typing import TYPE_CHECKING, Optional
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.stomp_utils import StompConnectionManager
+from rucio.common.types import InternalScope, LoggerFunction
+from rucio.core.monitor import MetricManager
+from rucio.core.rse import get_rse_id
+from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_replicas
+
+if TYPE_CHECKING:
+    from types import FrameType
+
+    from stomp import Connection
+    from stomp.utils import Frame
+
+logging.getLogger("stomp").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'cache-consumer'
+
+
+class AMQConsumer:
+    """
+    class Consumer
+    """
+
+    def __init__(
+            self,
+            broker: str,
+            conn: "Connection",
+            logger: "LoggerFunction"
+    ):
+        """
+        __init__
+        """
+        self.__broker = broker
+        self.__conn = conn
+        self.__logger = logger
+
+    @METRICS.count_it
+    def on_heartbeat_timeout(self) -> None:
+        self.__conn.disconnect()
+
+    @METRICS.count_it
+    def on_error(self, frame: "Frame") -> None:
+        """
+        on_error
+        """
+        self.__logger(logging.ERROR, 'Message receive error: [%s] %s' % (self.__broker, frame.body))
+
+    @METRICS.count_it
+    def on_message(self, frame: "Frame") -> None:
+        """
+        on_message
+        """
+        try:
+            msg = json.loads(frame.body)  # type: ignore
+            self.__logger(logging.DEBUG, 'Message received: %s ' % msg)
+            if isinstance(msg, dict) and 'operation' in msg.keys():
+                for f in msg['files']:
+                    f['scope'] = InternalScope(f['scope'])
+                if 'rse_id' in msg:
+                    rse_id = msg['rse_id']
+                else:
+                    rse_id = get_rse_id(rse=msg['rse'], vo=msg.get('vo', 'def'))
+
+                rse_vo_str = msg['rse']
+                if 'vo' in msg and msg['vo'] != 'def':
+                    rse_vo_str = '{} on {}'.format(rse_vo_str, msg['vo'])
+                if msg['operation'] == 'add_replicas':
+                    self.__logger(logging.INFO, 'add_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+                elif msg['operation'] == 'delete_replicas':
+                    self.__logger(logging.INFO, 'delete_replicas to RSE %s: %s ' % (rse_vo_str, str(msg['files'])))
+                    delete_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+            else:
+                self.__logger(logging.DEBUG, 'Check failed: %s %s '
+                              % (isinstance(msg, dict), 'operation' in msg.keys()))
+        except:
+            self.__logger(logging.ERROR, str(format_exc()))
+
+
+def consumer(id_: int, num_thread: int = 1) -> None:
+    """
+    Main loop to consume messages from the Rucio Cache producer.
+    """
+
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
+
+    logger(logging.INFO, 'Rucio Cache consumer starting')
+
+    brokers = config_get_list('messaging-cache', 'brokers')
+
+    use_ssl = config_get_bool('messaging-cache', 'use_ssl', default=True, raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-cache', 'username')
+        password = config_get('messaging-cache', 'password')
+    destination = config_get('messaging-cache', 'destination')
+    subscription_id = 'rucio-cache-messaging'
+
+    vhost = config_get('messaging-cache', 'broker_virtual_host', raise_exception=False)
+    port = config_get_int('messaging-cache', 'port')
+    reconnect_attempts = config_get_int('messaging-cache', 'reconnect_attempts', default=100)
+    ssl_key_file = config_get('messaging-cache', 'ssl_key_file', raise_exception=False)
+    ssl_cert_file = config_get('messaging-cache', 'ssl_cert_file', raise_exception=False)
+
+    stomp_conn_mngr = StompConnectionManager()
+    conns, _ = stomp_conn_mngr.re_configure(
+        brokers=brokers,
+        port=port,
+        use_ssl=use_ssl,
+        vhost=vhost,
+        reconnect_attempts=reconnect_attempts,
+        ssl_key_file=ssl_key_file,
+        ssl_cert_file=ssl_cert_file,
+        timeout=None,
+        logger=logger
+    )
+
+    logger(logging.INFO, 'consumer started')
+
+    while not GRACEFUL_STOP.is_set():
+        for conn in conns:
+            if not conn.is_connected():
+                host_port = conn.transport._Transport__host_and_ports[0]
+
+                logger(logging.INFO, 'connecting to %s' % host_port[0])
+                METRICS.counter('reconnect.{host}').labels(host=host_port[0]).inc()
+                conn.set_listener('rucio-cache-consumer', AMQConsumer(broker=host_port, conn=conn, logger=logger))
+                if not use_ssl:
+                    conn.connect(username, password)
+                else:
+                    conn.connect()
+
+                conn.subscribe(destination=destination, ack='auto', id=subscription_id)
+        time.sleep(1)
+
+    logger(logging.INFO, 'graceful stop requested')
+    stomp_conn_mngr.disconnect()
+    logger(logging.INFO, 'graceful stop done')
+
+
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(num_thread: int = 1) -> None:
+    """
+    Starts up the rucio cache consumer thread
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    logging.info('starting consumer thread')
+    threads = [threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread})
+               for i in range(0, num_thread)]
+
+    [t.start() for t in threads]
+
+    logging.info('waiting for interrupts')
+
+    # Interruptible joins require a timeout.
+    while threads[0].is_alive():
+        [t.join(timeout=3.14) for t in threads]

rucio-37.0.0rc4/lib/rucio/daemons/conveyor/receiver.py

@@ -0,0 +1,249 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import json
+import logging
+import socket
+import threading
+import time
+import traceback
+from typing import TYPE_CHECKING, Any, Optional
+
+import stomp
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.logging import setup_logging
+from rucio.common.policy import get_policy
+from rucio.core import request as request_core
+from rucio.core import transfer as transfer_core
+from rucio.core.monitor import MetricManager
+from rucio.daemons.common import HeartbeatHandler
+from rucio.db.sqla.session import transactional_session
+from rucio.transfertool.fts3 import FTS3CompletionMessageTransferStatusReport
+
+if TYPE_CHECKING:
+    from types import FrameType
+
+    from sqlalchemy.orm import Session
+    from stomp.utils import Frame
+
+    from rucio.common.types import LoggerFunction
+
+logging.getLogger("stomp").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'conveyor-receiver'
+
+
+class Receiver:
+
+    def __init__(
+            self,
+            broker: str,
+            id_: str,
+            total_threads: int,
+            transfer_stats_manager: request_core.TransferStatsManager,
+            all_vos: bool = False
+    ):
+        self.__all_vos = all_vos
+        self.__broker = broker
+        self.__id = id_
+        self.__total_threads = total_threads
+        self._transfer_stats_manager = transfer_stats_manager
+
+    @METRICS.count_it
+    def on_error(self, frame: "Frame") -> None:
+        logging.error('[%s] %s' % (self.__broker, frame.body))
+
+    @METRICS.count_it
+    def on_message(self, frame: "Frame") -> None:
+        msg = json.loads(frame.body)  # type: ignore
+
+        if not self.__all_vos:
+            if 'vo' not in msg or msg['vo'] != get_policy():
+                return
+
+        if 'job_metadata' in msg.keys() \
+           and isinstance(msg['job_metadata'], dict) \
+           and 'issuer' in msg['job_metadata'].keys() \
+           and str(msg['job_metadata']['issuer']) == 'rucio':
+
+            if 'job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE' or msg.get('job_multihop', False) is True):
+                METRICS.counter('message_rucio').inc()
+
+                self._perform_request_update(msg)
+
+    @transactional_session
+    def _perform_request_update(
+        self,
+        msg: dict[str, Any],
+        *,
+        session: Optional["Session"] = None,
+        logger: "LoggerFunction" = logging.log
+    ) -> None:
+        external_host = msg.get('endpnt', None)
+        request_id = msg['file_metadata'].get('request_id', None)
+        try:
+            tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host, request_id=request_id, fts_message=msg)
+            if tt_status_report.get_db_fields_to_update(session=session, logger=logger):  # type: ignore
+                logging.info('RECEIVED %s', tt_status_report)
+
+                ret = transfer_core.update_transfer_state(
+                    tt_status_report=tt_status_report,
+                    stats_manager=self._transfer_stats_manager,
+                    session=session,
+                    logger=logger,
+                )
+                if ret:
+                    METRICS.counter('update_request_state.{updated}').labels(updated=True).inc(delta=ret)
+                else:
+                    METRICS.counter('update_request_state.{updated}').labels(updated=False).inc()
+        except Exception:
+            logging.critical(traceback.format_exc())
+
+
+def receiver(
+        id_: str,
+        total_threads: int = 1,
+        all_vos: bool = False
+) -> None:
+    """
+    Main loop to consume messages from the FTS3 producer.
+    """
+
+    logging.info('receiver starting')
+
+    brokers_alias = []
+    brokers_resolved = []
+    try:
+        brokers_alias = config_get_list('messaging-fts3', 'brokers')
+    except Exception:
+        raise Exception('Could not load brokers from configuration')
+
+    logging.info('resolving broker dns alias: %s' % brokers_alias)
+
+    brokers_resolved = []
+    for broker in brokers_alias:
+        addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        brokers_resolved.extend(ai[4][0] for ai in addrinfos)
+
+    logging.info('brokers resolved to %s', brokers_resolved)
+
+    logging.info('checking authentication method')
+    use_ssl = True
+    try:
+        use_ssl = config_get_bool('messaging-fts3', 'use_ssl')
+    except:
+        logging.info('could not find use_ssl in configuration -- please update your rucio.cfg')
+
+    port = config_get_int('messaging-fts3', 'port')
+    vhost = config_get('messaging-fts3', 'broker_virtual_host', raise_exception=False)
+    if not use_ssl:
+        username = config_get('messaging-fts3', 'username')
+        password = config_get('messaging-fts3', 'password')
+        port = config_get_int('messaging-fts3', 'nonssl_port')
+
+    conns = []
+    for broker in brokers_resolved:
+        if not use_ssl:
+            logging.info('setting up username/password authentication: %s' % broker)
+        else:
+            logging.info('setting up ssl cert/key authentication: %s' % broker)
+        con = stomp.Connection12(host_and_ports=[(broker, port)],
+                                 vhost=vhost,
+                                 reconnect_attempts_max=999)
+        if use_ssl:
+            con.set_ssl(
+                key_file=config_get('messaging-fts3', 'ssl_key_file'),
+                cert_file=config_get('messaging-fts3', 'ssl_cert_file'),
+            )
+        conns.append(con)
+
+    logging.info('receiver started')
+
+    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30) as heartbeat_handler,
+          request_core.TransferStatsManager() as transfer_stats_manager):
+        while not GRACEFUL_STOP.is_set():
+
+            _, _, logger = heartbeat_handler.live()
+
+            for conn in conns:
+
+                if not conn.is_connected():
+                    logger(logging.INFO, 'connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
+                    METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0].split('.')[0]).inc()
+
+                    conn.set_listener(
+                        'rucio-messaging-fts3',
+                        Receiver(
+                            broker=conn.transport._Transport__host_and_ports[0],
+                            id_=id_,
+                            total_threads=total_threads,
+                            transfer_stats_manager=transfer_stats_manager,
+                            all_vos=all_vos
+                        ))
+                    if not use_ssl:
+                        conn.connect(username, password, wait=True)
+                    else:
+                        conn.connect(wait=True)
+                    conn.subscribe(destination=config_get('messaging-fts3', 'destination'),
+                                   id='rucio-messaging-fts3',
+                                   ack='auto')
+            time.sleep(1)
+
+        for conn in conns:
+            try:
+                conn.disconnect()
+            except Exception:
+                pass
+
+
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(
+        once: bool = False,
+        total_threads: int = 1
+) -> None:
+    """
+    Starts up the receiver thread
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    logging.info('starting receiver thread')
+    threads = [threading.Thread(target=receiver, kwargs={'id_': i,
+                                                         'total_threads': total_threads}) for i in range(0, total_threads)]
+
+    [thread.start() for thread in threads]
+
+    logging.info('waiting for interrupts')
+
+    # Interruptible joins require a timeout.
+    while threads:
+        threads = [thread.join(timeout=3.14) for thread in threads if thread and thread.is_alive()]