rucio 37.0.0rc1__py3-none-any.whl

rucio/daemons/bb8/nuclei_background_rebalance.py

@@ -0,0 +1,153 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to be used to background rebalance ATLAS t2 datadisks
+"""
+
+from sqlalchemy import and_, or_, select
+
+from rucio.common.constants import RseAttr
+from rucio.core.rse import get_rse_attribute, get_rse_usage
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.bb8.common import rebalance_rse
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RuleState
+from rucio.db.sqla.session import get_session
+
+tolerance = 0.15
+max_total_rebalance_volume = 200 * 1E12
+max_rse_rebalance_volume = 20 * 1E12
+min_total = 50 * 1E12
+total_rebalance_volume = 0
+
+
+# groupdisks
+def group_space(site: str) -> int:
+    """
+    groupdisks of given site
+    contributing to primaries
+    """
+    site_groupdisks = []
+    group_total = 0
+    try:
+        site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
+    except:
+        return group_total
+
+    for rse in site_groupdisks:
+        used = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used']
+        group_total += used
+
+    return group_total
+
+
+# Calculate the current ratios
+rses = parse_expression("(datapolicynucleus=true|tier=1)&type=DATADISK\\bb8-enabled=false")
+total_primary = 0
+total_secondary = 0
+total_total = 0
+global_ratio = float(0)
+for rse in rses:
+    site_name = get_rse_attribute(rse['id'], RseAttr.SITE)
+    rse['groupdisk'] = group_space(site_name)  # type: ignore (site_name could be None)
+    rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['primary'] += rse['groupdisk']
+    rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['total'] = get_rse_usage(rse_id=rse['id'], source='storage')[0]['total'] - get_rse_usage(rse_id=rse['id'], source='min_free_space')[0]['used']
+    rse['ratio'] = float(rse['primary']) / float(rse['total'])
+    total_primary += rse['primary']
+    total_secondary += rse['secondary']
+    total_total += float(rse['total'])
+    rse['receive_volume'] = 0  # Already rebalanced volume in this run
+
+global_ratio = float(total_primary) / float(total_total)
+print('Global ratio: %f' % (global_ratio))
+for rse in sorted(rses, key=lambda k: k['ratio']):
+    print('  %s (%f)' % (rse['rse'], rse['ratio']))
+
+rses_over_ratio = sorted([rse for rse in rses if rse['ratio'] > global_ratio + global_ratio * tolerance], key=lambda k: k['ratio'], reverse=True)
+rses_under_ratio = sorted([rse for rse in rses if rse['ratio'] < global_ratio - global_ratio * tolerance], key=lambda k: k['ratio'], reverse=False)
+
+session = get_session()
+stmt = select(
+    models.ReplicationRule.rse_expression
+).where(
+    and_(or_(models.ReplicationRule.state == RuleState.REPLICATING,
+             models.ReplicationRule.state == RuleState.STUCK),
+         models.ReplicationRule.comments == 'T2 Background rebalancing')
+).group_by(
+    models.ReplicationRule.rse_expression
+)
+# Excluding RSEs
+print('Excluding RSEs as destination which have active Background Rebalancing rules:')
+for rse in session.execute(stmt).all():
+    print('  %s' % (rse[0]))
+    for des in rses_under_ratio:
+        des_as_expr = des['rse']
+        if des_as_expr == rse[0]:
+            rses_under_ratio.remove(des)
+            break
+
+print('Excluding RSEs as destination which are too small by size:')
+for des in rses_under_ratio:
+    if des['total'] < min_total:
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are too small by size:')
+for src in rses_over_ratio:
+    if src['total'] < min_total:
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+print('Excluding RSEs as destinations which are blocklisted:')
+for des in rses_under_ratio:
+    if not (des['availability_read'] and des['availability_write'] and des['availability_delete']):
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are blocklisted:')
+for src in rses_over_ratio:
+    if not (src['availability_read'] and src['availability_write'] and src['availability_delete']):
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+# Loop over RSEs over the ratio
+for source_rse in rses_over_ratio:
+
+    # The volume that would be rebalanced, not real availability of the data:
+    available_source_rebalance_volume = int((source_rse['primary'] - global_ratio * source_rse['secondary']) / (global_ratio + 1))
+    if available_source_rebalance_volume > max_rse_rebalance_volume:
+        available_source_rebalance_volume = max_rse_rebalance_volume
+    if available_source_rebalance_volume > max_total_rebalance_volume - total_rebalance_volume:
+        available_source_rebalance_volume = max_total_rebalance_volume - total_rebalance_volume
+
+    # Select a target:
+    for destination_rse in rses_under_ratio:
+        if available_source_rebalance_volume > 0:
+            if destination_rse['receive_volume'] >= max_rse_rebalance_volume:
+                continue
+            available_target_rebalance_volume = max_rse_rebalance_volume - destination_rse['receive_volume']
+            if available_target_rebalance_volume >= available_source_rebalance_volume:
+                available_target_rebalance_volume = available_source_rebalance_volume
+
+            vo_str = 'on VO {}'.format(destination_rse['vo']) if destination_rse['vo'] != 'def' else 'def'
+            print('Rebalance %dTB from %s(%f) to %s(%f)%s' % (available_target_rebalance_volume / 1E12, source_rse['rse'], source_rse['ratio'], destination_rse['rse'], destination_rse['ratio'], vo_str))
+            expr = destination_rse['rse']
+            rebalance_rse(rse_id=source_rse['id'], max_bytes=available_target_rebalance_volume, dry_run=False, comment='Nuclei Background rebalancing', force_expression=expr)
+
+            destination_rse['receive_volume'] += available_target_rebalance_volume
+            total_rebalance_volume += available_target_rebalance_volume
+            available_source_rebalance_volume -= available_target_rebalance_volume

rucio/daemons/bb8/t2_background_rebalance.py

@@ -0,0 +1,153 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to be used to background rebalance ATLAS t2 datadisks
+"""
+
+from sqlalchemy import and_, or_, select
+
+from rucio.common.constants import RseAttr
+from rucio.core.rse import get_rse_attribute, get_rse_usage
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.bb8.common import rebalance_rse
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RuleState
+from rucio.db.sqla.session import get_session
+
+tolerance = 0.1
+max_total_rebalance_volume = 200 * 1E12
+max_rse_rebalance_volume = 20 * 1E12
+min_total = 50 * 1E12
+total_rebalance_volume = 0
+
+
+# groupdisks
+def group_space(site: str) -> int:
+    """
+    groupdisks of given site
+    contributing to primaries
+    """
+    site_groupdisks = []
+    group_total = 0
+    try:
+        site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
+    except:
+        return group_total
+
+    for rse in site_groupdisks:
+        used = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used']
+        group_total += used
+
+    return group_total
+
+
+# Calculate the current ratios
+rses = parse_expression("datapolicynucleus=false&tier=2&type=DATADISK\\bb8-enabled=false")
+total_primary = 0
+total_secondary = 0
+total_total = 0
+global_ratio = float(0)
+for rse in rses:
+    site_name = get_rse_attribute(rse['id'], RseAttr.SITE)
+    rse['groupdisk'] = group_space(site_name)  # type: ignore (site_name could be None)
+    rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['primary'] += rse['groupdisk']
+    rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['total'] = get_rse_usage(rse_id=rse['id'], source='storage')[0]['total'] - get_rse_usage(rse_id=rse['id'], source='min_free_space')[0]['used']
+    rse['ratio'] = float(rse['primary']) / float(rse['total'])
+    total_primary += rse['primary']
+    total_secondary += rse['secondary']
+    total_total += float(rse['total'])
+    rse['receive_volume'] = 0  # Already rebalanced volume in this run
+
+global_ratio = float(total_primary) / float(total_total)
+print('Global ratio: %f' % (global_ratio))
+for rse in sorted(rses, key=lambda k: k['ratio']):
+    print('  %s (%f)' % (rse['rse'], rse['ratio']))
+
+rses_over_ratio = sorted([rse for rse in rses if rse['ratio'] > global_ratio + global_ratio * tolerance], key=lambda k: k['ratio'], reverse=True)
+rses_under_ratio = sorted([rse for rse in rses if rse['ratio'] < global_ratio - global_ratio * tolerance], key=lambda k: k['ratio'], reverse=False)
+
+session = get_session()
+stmt = select(
+    models.ReplicationRule.rse_expression
+).where(
+    and_(or_(models.ReplicationRule.state == RuleState.REPLICATING,
+             models.ReplicationRule.state == RuleState.STUCK),
+         models.ReplicationRule.comments == 'T2 Background rebalancing')
+).group_by(
+    models.ReplicationRule.rse_expression
+)
+
+# Excluding RSEs
+print('Excluding RSEs as destination which have active Background Rebalancing rules:')
+for rse in session.execute(stmt).all():
+    print('  %s' % (rse[0]))
+    for des in rses_under_ratio:
+        des_as_expr = des['rse']
+        if des_as_expr == rse[0]:
+            rses_under_ratio.remove(des)
+            break
+
+print('Excluding RSEs as destination which are too small by size:')
+for des in rses_under_ratio:
+    if des['total'] < min_total:
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are too small by size:')
+for src in rses_over_ratio:
+    if src['total'] < min_total:
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+print('Excluding RSEs as desetinations which are blocklisted:')
+for des in rses_under_ratio:
+    if not (des['availability_read'] and des['availability_write'] and des['availability_delete']):
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are blocklisted:')
+for src in rses_over_ratio:
+    if not (src['availability_read'] and src['availability_write'] and src['availability_delete']):
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+# Loop over RSEs over the ratio
+for source_rse in rses_over_ratio:
+
+    # The volume that would be rebalanced, not real availability of the data:
+    available_source_rebalance_volume = int((source_rse['primary'] - global_ratio * source_rse['secondary']) / (global_ratio + 1))
+    if available_source_rebalance_volume > max_rse_rebalance_volume:
+        available_source_rebalance_volume = max_rse_rebalance_volume
+    if available_source_rebalance_volume > max_total_rebalance_volume - total_rebalance_volume:
+        available_source_rebalance_volume = max_total_rebalance_volume - total_rebalance_volume
+
+    # Select a target:
+    for destination_rse in rses_under_ratio:
+        if available_source_rebalance_volume > 0:
+            if destination_rse['receive_volume'] >= max_rse_rebalance_volume:
+                continue
+            available_target_rebalance_volume = max_rse_rebalance_volume - destination_rse['receive_volume']
+            if available_target_rebalance_volume >= available_source_rebalance_volume:
+                available_target_rebalance_volume = available_source_rebalance_volume
+
+            print('Rebalance %dTB from %s(%f) to %s(%f)' % (available_target_rebalance_volume / 1E12, source_rse['rse'], source_rse['ratio'], destination_rse['rse'], destination_rse['ratio']))
+            expr = destination_rse['rse']
+            rebalance_rse(rse_id=source_rse['id'], max_bytes=available_target_rebalance_volume, dry_run=False, comment='T2 Background rebalancing', force_expression=expr)
+
+            destination_rse['receive_volume'] += available_target_rebalance_volume
+            total_rebalance_volume += available_target_rebalance_volume
+            available_source_rebalance_volume -= available_target_rebalance_volume

rucio/daemons/cache/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/cache/consumer.py

@@ -0,0 +1,133 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Cache consumer is a daemon to retrieve rucio cache operation information to synchronize rucio catalog.
+"""
+
+import json
+import logging
+import threading
+import time
+from traceback import format_exc
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import formatted_logger, setup_logging
+from rucio.common.stomp_utils import ListenerBase, StompConnectionManager
+from rucio.common.types import InternalScope, LoggerFunction
+from rucio.core.monitor import MetricManager
+from rucio.core.rse import get_rse_id
+from rucio.core.volatile_replica import add_volatile_replicas, delete_volatile_replicas
+
+if TYPE_CHECKING:
+    from types import FrameType
+
+    from stomp.utils import Frame
+
+logging.getLogger("stomp").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'cache-consumer'
+
+
+class AMQConsumer(ListenerBase):
+    """
+    class Consumer
+    """
+
+    @METRICS.count_it
+    def on_message(self, frame: "Frame") -> None:
+        """
+        on_message
+        """
+        try:
+            msg = json.loads(frame.body)  # type: ignore
+            self._logger(logging.DEBUG, 'Message received: %s', msg)
+            if isinstance(msg, dict) and 'operation' in msg.keys():
+                for f in msg['files']:
+                    f['scope'] = InternalScope(f['scope'])
+                if 'rse_id' in msg:
+                    rse_id = msg['rse_id']
+                else:
+                    rse_id = get_rse_id(rse=msg['rse'], vo=msg.get('vo', 'def'))
+
+                rse_vo_str = msg['rse']
+                if 'vo' in msg and msg['vo'] != 'def':
+                    rse_vo_str = f"{rse_vo_str} on {msg['vo']}"
+                if msg['operation'] == 'add_replicas':
+                    self._logger(logging.INFO, "add_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
+                    add_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+                elif msg['operation'] == 'delete_replicas':
+                    self._logger(logging.INFO, "delete_replicas to RSE %s: %s", rse_vo_str, str(msg['files']))
+                    delete_volatile_replicas(rse_id=rse_id, replicas=msg['files'])
+            else:
+                self._logger(logging.DEBUG, 'Check failed: %s %s', isinstance(msg, dict), "operation" in msg.keys())
+        except:
+            self._logger(logging.ERROR, str(format_exc()))
+
+
+def consumer(id_: int, num_thread: int = 1, logger: LoggerFunction = logging.log) -> None:
+    """
+    Main loop to consume messages from the Rucio Cache producer.
+    """
+    logger(logging.INFO, 'Rucio Cache consumer starting')
+
+    conn_mgr = StompConnectionManager(config_section='messaging-cache', logger=logger)
+
+    logger(logging.INFO, 'consumer started')
+
+    conn_mgr.set_listener_factory('rucio-cache-consumer', AMQConsumer, heartbeats=conn_mgr.config.heartbeats)
+
+    while not GRACEFUL_STOP.is_set():
+
+        conn_mgr.subscribe(id_='rucio-cache-messaging', ack='auto')
+        time.sleep(1)
+
+    logger(logging.INFO, 'graceful stop requested')
+    conn_mgr.disconnect()
+    logger(logging.INFO, 'graceful stop done')
+
+
+def stop(signum: "int | None" = None, frame: "FrameType | None" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(num_thread: int = 1) -> None:
+    """
+    Starts up the rucio cache consumer thread
+    """
+    setup_logging(process_name=DAEMON_NAME)
+    logger = formatted_logger(logging.log, DAEMON_NAME + ' %s')
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    logger(logging.INFO, 'starting consumer thread')
+    threads = []
+    for i in range(num_thread):
+        con_thread = threading.Thread(target=consumer, kwargs={'id_': i, 'num_thread': num_thread, 'logger': logger})
+        con_thread.start()
+        threads.append(con_thread)
+
+    logger(logging.INFO, 'waiting for interrupts')
+
+    while [thread.join(timeout=3.14) for thread in threads if thread.is_alive()]:
+        pass