rucio 32.8.6__py3-none-any.whl

rucio/daemons/bb8/nuclei_background_rebalance.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to be used to background rebalance ATLAS t2 datadisks
+"""
+
+from sqlalchemy import or_
+
+from rucio.core.rse import get_rse_usage, get_rse_attribute
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.bb8.common import rebalance_rse
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RuleState
+from rucio.db.sqla.session import get_session
+
+tolerance = 0.15
+max_total_rebalance_volume = 200 * 1E12
+max_rse_rebalance_volume = 20 * 1E12
+min_total = 50 * 1E12
+total_rebalance_volume = 0
+
+
+# groupdisks
+def group_space(site):
+    """
+    groupdisks of given site
+    contributing to primaries
+    """
+    site_groupdisks = []
+    group_total = 0
+    try:
+        site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
+    except:
+        return group_total
+
+    for rse in site_groupdisks:
+        used = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used']
+        group_total += used
+
+    return group_total
+
+
+# Calculate the current ratios
+rses = parse_expression("(datapolicynucleus=true|tier=1)&type=DATADISK\\bb8-enabled=false")
+total_primary = 0
+total_secondary = 0
+total_total = 0
+global_ratio = float(0)
+for rse in rses:
+    site_name = get_rse_attribute(rse['id'], 'site')
+    rse['groupdisk'] = group_space(site_name)
+    rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['primary'] += rse['groupdisk']
+    rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['total'] = get_rse_usage(rse_id=rse['id'], source='storage')[0]['total'] - get_rse_usage(rse_id=rse['id'], source='min_free_space')[0]['used']
+    rse['ratio'] = float(rse['primary']) / float(rse['total'])
+    total_primary += rse['primary']
+    total_secondary += rse['secondary']
+    total_total += float(rse['total'])
+    rse['receive_volume'] = 0  # Already rebalanced volume in this run
+
+global_ratio = float(total_primary) / float(total_total)
+print('Global ratio: %f' % (global_ratio))
+for rse in sorted(rses, key=lambda k: k['ratio']):
+    print('  %s (%f)' % (rse['rse'], rse['ratio']))
+
+rses_over_ratio = sorted([rse for rse in rses if rse['ratio'] > global_ratio + global_ratio * tolerance], key=lambda k: k['ratio'], reverse=True)
+rses_under_ratio = sorted([rse for rse in rses if rse['ratio'] < global_ratio - global_ratio * tolerance], key=lambda k: k['ratio'], reverse=False)
+
+session = get_session()
+active_rses = session.query(models.ReplicationRule.rse_expression).filter(or_(models.ReplicationRule.state == RuleState.REPLICATING, models.ReplicationRule.state == RuleState.STUCK),
+                                                                          models.ReplicationRule.comments == 'T2 Background rebalancing').group_by(models.ReplicationRule.rse_expression).all()
+
+# Excluding RSEs
+print('Excluding RSEs as destination which have active Background Rebalancing rules:')
+for rse in active_rses:
+    print('  %s' % (rse[0]))
+    for des in rses_under_ratio:
+        des_as_expr = des['rse']
+        if des_as_expr == rse[0]:
+            rses_under_ratio.remove(des)
+            break
+
+print('Excluding RSEs as destination which are too small by size:')
+for des in rses_under_ratio:
+    if des['total'] < min_total:
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are too small by size:')
+for src in rses_over_ratio:
+    if src['total'] < min_total:
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+print('Excluding RSEs as destinations which are blocklisted:')
+for des in rses_under_ratio:
+    if not (des['availability_read'] and des['availability_write'] and des['availability_delete']):
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are blocklisted:')
+for src in rses_over_ratio:
+    if not (src['availability_read'] and src['availability_write'] and src['availability_delete']):
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+# Loop over RSEs over the ratio
+for source_rse in rses_over_ratio:
+
+    # The volume that would be rebalanced, not real availability of the data:
+    available_source_rebalance_volume = int((source_rse['primary'] - global_ratio * source_rse['secondary']) / (global_ratio + 1))
+    if available_source_rebalance_volume > max_rse_rebalance_volume:
+        available_source_rebalance_volume = max_rse_rebalance_volume
+    if available_source_rebalance_volume > max_total_rebalance_volume - total_rebalance_volume:
+        available_source_rebalance_volume = max_total_rebalance_volume - total_rebalance_volume
+
+    # Select a target:
+    for destination_rse in rses_under_ratio:
+        if available_source_rebalance_volume > 0:
+            if destination_rse['receive_volume'] >= max_rse_rebalance_volume:
+                continue
+            available_target_rebalance_volume = max_rse_rebalance_volume - destination_rse['receive_volume']
+            if available_target_rebalance_volume >= available_source_rebalance_volume:
+                available_target_rebalance_volume = available_source_rebalance_volume
+
+            vo_str = 'on VO {}'.format(destination_rse['vo']) if destination_rse['vo'] != 'def' else 'def'
+            print('Rebalance %dTB from %s(%f) to %s(%f)%s' % (available_target_rebalance_volume / 1E12, source_rse['rse'], source_rse['ratio'], destination_rse['rse'], destination_rse['ratio'], vo_str))
+            expr = destination_rse['rse']
+            rebalance_rse(rse_id=source_rse['id'], max_bytes=available_target_rebalance_volume, dry_run=False, comment='Nuclei Background rebalancing', force_expression=expr)
+
+            destination_rse['receive_volume'] += available_target_rebalance_volume
+            total_rebalance_volume += available_target_rebalance_volume
+            available_source_rebalance_volume -= available_target_rebalance_volume

rucio/daemons/bb8/t2_background_rebalance.py

@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to be used to background rebalance ATLAS t2 datadisks
+"""
+
+from sqlalchemy import or_
+
+from rucio.core.rse import get_rse_usage, get_rse_attribute
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.bb8.common import rebalance_rse
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RuleState
+from rucio.db.sqla.session import get_session
+
+tolerance = 0.1
+max_total_rebalance_volume = 200 * 1E12
+max_rse_rebalance_volume = 20 * 1E12
+min_total = 50 * 1E12
+total_rebalance_volume = 0
+
+
+# groupdisks
+def group_space(site):
+    """
+    groupdisks of given site
+    contributing to primaries
+    """
+    site_groupdisks = []
+    group_total = 0
+    try:
+        site_groupdisks = parse_expression('site=%s&spacetoken=ATLASDATADISK&type=GROUPDISK' % site)
+    except:
+        return group_total
+
+    for rse in site_groupdisks:
+        used = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used']
+        group_total += used
+
+    return group_total
+
+
+# Calculate the current ratios
+rses = parse_expression("datapolicynucleus=false&tier=2&type=DATADISK\\bb8-enabled=false")
+total_primary = 0
+total_secondary = 0
+total_total = 0
+global_ratio = float(0)
+for rse in rses:
+    site_name = get_rse_attribute(rse['id'], 'site')
+    rse['groupdisk'] = group_space(site_name)
+    rse['primary'] = get_rse_usage(rse_id=rse['id'], source='rucio')[0]['used'] - get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['primary'] += rse['groupdisk']
+    rse['secondary'] = get_rse_usage(rse_id=rse['id'], source='expired')[0]['used']
+    rse['total'] = get_rse_usage(rse_id=rse['id'], source='storage')[0]['total'] - get_rse_usage(rse_id=rse['id'], source='min_free_space')[0]['used']
+    rse['ratio'] = float(rse['primary']) / float(rse['total'])
+    total_primary += rse['primary']
+    total_secondary += rse['secondary']
+    total_total += float(rse['total'])
+    rse['receive_volume'] = 0  # Already rebalanced volume in this run
+
+global_ratio = float(total_primary) / float(total_total)
+print('Global ratio: %f' % (global_ratio))
+for rse in sorted(rses, key=lambda k: k['ratio']):
+    print('  %s (%f)' % (rse['rse'], rse['ratio']))
+
+rses_over_ratio = sorted([rse for rse in rses if rse['ratio'] > global_ratio + global_ratio * tolerance], key=lambda k: k['ratio'], reverse=True)
+rses_under_ratio = sorted([rse for rse in rses if rse['ratio'] < global_ratio - global_ratio * tolerance], key=lambda k: k['ratio'], reverse=False)
+
+session = get_session()
+active_rses = session.query(models.ReplicationRule.rse_expression).filter(or_(models.ReplicationRule.state == RuleState.REPLICATING, models.ReplicationRule.state == RuleState.STUCK),
+                                                                          models.ReplicationRule.comments == 'T2 Background rebalancing').group_by(models.ReplicationRule.rse_expression).all()
+
+# Excluding RSEs
+print('Excluding RSEs as destination which have active Background Rebalancing rules:')
+for rse in active_rses:
+    print('  %s' % (rse[0]))
+    for des in rses_under_ratio:
+        des_as_expr = des['rse']
+        if des_as_expr == rse[0]:
+            rses_under_ratio.remove(des)
+            break
+
+print('Excluding RSEs as destination which are too small by size:')
+for des in rses_under_ratio:
+    if des['total'] < min_total:
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are too small by size:')
+for src in rses_over_ratio:
+    if src['total'] < min_total:
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+print('Excluding RSEs as desetinations which are blocklisted:')
+for des in rses_under_ratio:
+    if not (des['availability_read'] and des['availability_write'] and des['availability_delete']):
+        print('  %s' % (des['rse']))
+        rses_under_ratio.remove(des)
+
+print('Excluding RSEs as sources which are blocklisted:')
+for src in rses_over_ratio:
+    if not (src['availability_read'] and src['availability_write'] and src['availability_delete']):
+        print('  %s' % (src['rse']))
+        rses_over_ratio.remove(src)
+
+# Loop over RSEs over the ratio
+for source_rse in rses_over_ratio:
+
+    # The volume that would be rebalanced, not real availability of the data:
+    available_source_rebalance_volume = int((source_rse['primary'] - global_ratio * source_rse['secondary']) / (global_ratio + 1))
+    if available_source_rebalance_volume > max_rse_rebalance_volume:
+        available_source_rebalance_volume = max_rse_rebalance_volume
+    if available_source_rebalance_volume > max_total_rebalance_volume - total_rebalance_volume:
+        available_source_rebalance_volume = max_total_rebalance_volume - total_rebalance_volume
+
+    # Select a target:
+    for destination_rse in rses_under_ratio:
+        if available_source_rebalance_volume > 0:
+            if destination_rse['receive_volume'] >= max_rse_rebalance_volume:
+                continue
+            available_target_rebalance_volume = max_rse_rebalance_volume - destination_rse['receive_volume']
+            if available_target_rebalance_volume >= available_source_rebalance_volume:
+                available_target_rebalance_volume = available_source_rebalance_volume
+
+            print('Rebalance %dTB from %s(%f) to %s(%f)' % (available_target_rebalance_volume / 1E12, source_rse['rse'], source_rse['ratio'], destination_rse['rse'], destination_rse['ratio']))
+            expr = destination_rse['rse']
+            rebalance_rse(rse_id=source_rse['id'], max_bytes=available_target_rebalance_volume, dry_run=False, comment='T2 Background rebalancing', force_expression=expr)
+
+            destination_rse['receive_volume'] += available_target_rebalance_volume
+            total_rebalance_volume += available_target_rebalance_volume
+            available_source_rebalance_volume -= available_target_rebalance_volume

rucio/daemons/c3po/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/c3po/algorithms/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/c3po/algorithms/simple.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from operator import itemgetter
+from random import shuffle
+
+from rucio.common.exception import DataIdentifierNotFound
+from rucio.core.did import get_did
+from rucio.core.replica import list_dataset_replicas
+from rucio.daemons.c3po.collectors.agis import MappingCollector
+from rucio.daemons.c3po.collectors.workload import WorkloadCollector
+from rucio.db.sqla.constants import ReplicaState
+
+
+class PlacementAlgorithm:
+    def __init__(self):
+        self._mc = MappingCollector()
+        self._wc = WorkloadCollector()
+        self.__setup_penalties()
+
+    def __setup_penalties(self):
+        self._penalties = {}
+        for panda_site in self._wc.get_sites():
+            site = self._mc.panda_to_site(panda_site)
+            self._penalties[site] = 0.1
+
+    def __update_penalties(self):
+        for site, penalty in self._penalties.items():
+            if penalty > 0.1:
+                self._penalties[site] = penalty - 0.1
+
+    def place(self, did):
+        self.__update_penalties()
+        decision = {'did': '{}:{}'.format(did['scope'].internal, did['name'])}
+        try:
+            meta = get_did(did[0], did[1])
+        except DataIdentifierNotFound:
+            decision['error_reason'] = 'did does not exist'
+            return decision
+        if meta['length'] is None:
+            meta['length'] = 0
+        if meta['bytes'] is None:
+            meta['bytes'] = 0
+        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
+
+        decision['length'] = meta['length']
+        decision['bytes'] = meta['bytes']
+
+        available_rses = []
+        available_sites = []
+        reps = list_dataset_replicas(did[0], did[1])
+
+        num_reps = 0
+        for rep in reps:
+            if rep['state'] == ReplicaState.AVAILABLE:
+                available_rses.append(rep['rse'])
+                available_sites.append(self._mc.ddm_to_site(rep['rse']))
+                num_reps += 1
+
+        decision['replica_rses'] = available_rses
+        decision['num_replicas'] = num_reps
+        if num_reps >= 5:
+            decision['error_reason'] = 'more than 4 replicas already exist'
+            return decision
+
+        site_ratios = {}
+        site_job_info = {}
+        for panda_site in self._wc.get_sites():
+            site = self._mc.panda_to_site(panda_site)
+            job_info = self._wc.get_job_info(panda_site)
+            ratio = float(job_info[0]) / (float(job_info[1]) + float(job_info[2]) / 2)
+            penalty = self._penalties[site]
+            site_ratios[site] = ratio * penalty
+            site_job_info[site] = (job_info, penalty)
+
+        decision['site_ratios'] = site_ratios
+        decision['site_job_info'] = site_job_info
+        picked_site = None
+        picked_rse = None
+
+        for site, _ in sorted(site_ratios.items(), key=itemgetter(1)):
+            if site in available_sites:
+                continue
+            rses_for_site = self._mc.site_to_ddm(site)
+            if rses_for_site is None:
+                continue
+
+            for rse in rses_for_site:
+                if 'DATADISK' in rse:
+                    picked_rse = rse
+                    picked_site = site
+                    break
+            if picked_rse:
+                break
+
+        if picked_rse is None:
+            decision['error_reason'] = 'could not pick RSE'
+            return decision
+
+        decision['destination_rse'] = picked_rse
+        if picked_site:
+            self._penalties[site] = 1
+
+        picked_source = None
+        shuffle(available_rses)
+        for rse in available_rses:
+            if 'TAPE' in rse:
+                continue
+            picked_source = rse
+            break
+
+        if picked_source is None:
+            picked_source = available_rses[0]
+
+        decision['source_rse'] = picked_source
+        logging.debug("Picked %s as source and %s as destination RSE" % (picked_source, picked_rse))
+
+        return decision

rucio/daemons/c3po/algorithms/t2_free_space.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from operator import itemgetter
+
+from rucio.common.config import config_get, config_get_int
+from rucio.common.exception import DataIdentifierNotFound
+from rucio.core.did import get_did
+from rucio.core.replica import list_dataset_replicas
+from rucio.core.rse import list_rse_attributes, get_rse_name
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
+from rucio.daemons.c3po.utils.dataset_cache import DatasetCache
+from rucio.daemons.c3po.utils.popularity import get_popularity
+from rucio.db.sqla.constants import ReplicaState
+
+
+class PlacementAlgorithm:
+    """
+    Placement algorithm that focusses on free space on T2 DATADISK RSEs.
+    """
+    def __init__(self):
+        self._fsc = FreeSpaceCollector()
+        self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
+
+        rse_expr = "tier=2&type=DATADISK"
+        rse_attrs = parse_expression(rse_expr)
+
+        self._rses = []
+        for rse in rse_attrs:
+            self._rses.append(rse['id'])
+
+        self.__setup_penalties()
+
+    def __setup_penalties(self):
+        self._penalties = {}
+        for rse_id in self._rses:
+            self._penalties[rse_id] = 1.0
+
+    def __update_penalties(self):
+        for rse_id, penalty in self._penalties.items():
+            if penalty > 1.0:
+                self._penalties[rse_id] = penalty - 1
+
+    def place(self, did):
+        self.__update_penalties()
+        decision = {'did': '{}:{}'.format(did[0].internal, did[1])}
+        if (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
+            decision['error_reason'] = 'not a data or mc dataset'
+            return decision
+
+        try:
+            meta = get_did(did[0], did[1])
+        except DataIdentifierNotFound:
+            decision['error_reason'] = 'did does not exist'
+            return decision
+        if meta['length'] is None:
+            meta['length'] = 0
+        if meta['bytes'] is None:
+            meta['bytes'] = 0
+        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
+
+        decision['length'] = meta['length']
+        decision['bytes'] = meta['bytes']
+
+        last_accesses = self._dc.get_did(did)
+        self._dc.add_did(did)
+
+        decision['last_accesses'] = last_accesses
+
+        pop = get_popularity(did)
+        decision['popularity'] = pop or 0.0
+
+        if (last_accesses < 5) and (pop < 10.0):
+            decision['error_reason'] = 'did not popular enough'
+            return decision
+
+        free_rses = self._rses
+        available_reps = []
+        reps = list_dataset_replicas(did[0], did[1])
+        num_reps = 0
+        for rep in reps:
+            rse_attr = list_rse_attributes(rep['rse_id'])
+            if 'type' not in rse_attr:
+                continue
+            if rse_attr['type'] != 'DATADISK':
+                continue
+            if rep['state'] == ReplicaState.AVAILABLE:
+                if rep['rse_id'] in free_rses:
+                    free_rses.remove(rep['rse_id'])
+                available_reps.append(rep['rse_id'])
+                num_reps += 1
+
+        decision['replica_rses'] = available_reps
+        decision['num_replicas'] = num_reps
+        if num_reps >= 5:
+            decision['error_reason'] = 'more than 4 replicas already exist'
+            return decision
+
+        rse_ratios = {}
+        space_info = self._fsc.get_rse_space()
+        for rse_id in free_rses:
+            rse_space = space_info[rse_id]
+            penalty = self._penalties[rse_id]
+            rse_ratios[rse_id] = float(rse_space['free']) / float(rse_space['total']) * 100.0 / penalty
+
+        sorted_rses = sorted(rse_ratios.items(), key=itemgetter(1), reverse=True)
+        decision['destination_rse'] = get_rse_name(rse_id=sorted_rses[0][0])
+        decision['rse_ratios'] = sorted_rses
+        self._penalties[sorted_rses[0][0]] = 10.0
+
+        return decision