rucio 32.8.6__py3-none-any.whl

rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from operator import itemgetter
+
+from rucio.common.config import config_get, config_get_int
+from rucio.common.exception import DataIdentifierNotFound
+from rucio.core.did import get_did
+from rucio.core.replica import list_dataset_replicas
+from rucio.core.rse import list_rse_attributes, get_rse_name
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
+from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache
+from rucio.daemons.c3po.utils.popularity import get_popularity
+from rucio.db.sqla.constants import ReplicaState
+
+
+class PlacementAlgorithm:
+    """
+    Placement algorithm that focusses on free space on T2 DATADISK RSEs.
+    """
+    def __init__(self):
+        self._fsc = FreeSpaceCollector()
+        self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
+
+        rse_expr = "tier=2&type=DATADISK"
+        rse_attrs = parse_expression(rse_expr)
+
+        self._rses = []
+        for rse in rse_attrs:
+            self._rses.append(rse['id'])
+
+        self.__setup_penalties()
+
+    def __setup_penalties(self):
+        self._penalties = {}
+        for rse_id in self._rses:
+            self._penalties[rse_id] = 1.0
+
+    def __update_penalties(self):
+        for rse_id, penalty in self._penalties.items():
+            if penalty > 1.0:
+                self._penalties[rse_id] = penalty - 1
+
+    def place(self, did):
+        self.__update_penalties()
+        decision = {'did': '{}:{}'.format(did[0].internal, did[1])}
+        if (self._added_cache.check_dataset(decision['did'])):
+            decision['error_reason'] = 'already added replica for this did in the last 24h'
+            return decision
+
+        if (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
+            decision['error_reason'] = 'not a data or mc dataset'
+            return decision
+
+        try:
+            meta = get_did(did[0], did[1])
+        except DataIdentifierNotFound:
+            decision['error_reason'] = 'did does not exist'
+            return decision
+        if meta['length'] is None:
+            meta['length'] = 0
+        if meta['bytes'] is None:
+            meta['bytes'] = 0
+        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
+
+        decision['length'] = meta['length']
+        decision['bytes'] = meta['bytes']
+
+        pop = get_popularity(did)
+        decision['popularity'] = pop or 0.0
+
+        if (pop < 10.0):
+            decision['error_reason'] = 'did not popular enough'
+            return decision
+
+        free_rses = self._rses
+        available_reps = []
+        reps = list_dataset_replicas(did[0], did[1])
+        num_reps = 0
+        for rep in reps:
+            rse_attr = list_rse_attributes(rse_id=rep['rse_id'])
+            if 'type' not in rse_attr:
+                continue
+            if rse_attr['type'] != 'DATADISK':
+                continue
+            if rep['state'] == ReplicaState.AVAILABLE:
+                if rep['rse_id'] in free_rses:
+                    free_rses.remove(rep['rse_id'])
+                available_reps.append(rep['rse_id'])
+                num_reps += 1
+
+        decision['replica_rses'] = available_reps
+        decision['num_replicas'] = num_reps
+
+        if num_reps >= 5:
+            decision['error_reason'] = 'more than 4 replicas already exist'
+            return decision
+
+        rse_ratios = {}
+        space_info = self._fsc.get_rse_space()
+        for rse_id in free_rses:
+            rse_space = space_info[rse_id]
+            penalty = self._penalties[rse_id]
+            rse_ratios[rse_id] = float(rse_space['free']) / float(rse_space['total']) * 100.0 / penalty
+
+        sorted_rses = sorted(rse_ratios.items(), key=itemgetter(1), reverse=True)
+        decision['destination_rse'] = get_rse_name(sorted_rses[0][0])
+        decision['rse_ratios'] = sorted_rses
+        self._penalties[sorted_rses[0][0]] = 10.0
+
+        self._added_cache.add_dataset(decision['did'])
+
+        return decision

rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py

@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from operator import itemgetter
+
+from rucio.common.config import config_get, config_get_int
+from rucio.common.exception import DataIdentifierNotFound
+from rucio.core.did import get_did
+from rucio.core.replica import list_dataset_replicas
+from rucio.core.rse import list_rse_attributes, get_rse, get_rse_name
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
+from rucio.daemons.c3po.collectors.network_metrics import NetworkMetricsCollector
+from rucio.daemons.c3po.utils.dataset_cache import DatasetCache
+from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache
+from rucio.daemons.c3po.utils.popularity import get_popularity
+from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
+from rucio.db.sqla.constants import ReplicaState
+
+
+class PlacementAlgorithm:
+    """
+    Placement algorithm that focusses on free space on T2 DATADISK RSEs. It incorporates network metrics for placement decisions.
+    """
+    def __init__(self, datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas):
+        self._fsc = FreeSpaceCollector()
+        self._nmc = NetworkMetricsCollector()
+        self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
+        self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
+        self._added_bytes = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_bytes_")
+        self._added_files = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_files_")
+
+        self._datatypes = datatypes.split(',')
+        self._dest_rse_expr = dest_rse_expr
+        self._max_bytes_hour = max_bytes_hour
+        self._max_files_hour = max_files_hour
+        self._max_bytes_hour_rse = max_bytes_hour_rse
+        self._max_files_hour_rse = max_files_hour_rse
+        self._min_popularity = min_popularity
+        self._min_recent_requests = min_recent_requests
+        self._max_replicas = max_replicas
+
+        rses = parse_expression(self._dest_rse_expr)
+
+        self._rses = {}
+        self._sites = {}
+        for rse in rses:
+            rse_attrs = list_rse_attributes(rse_id=rse['id'])
+            rse_attrs['rse_id'] = rse['id']
+            self._rses[rse['id']] = rse_attrs
+            self._sites[rse_attrs['site']] = rse_attrs
+
+        self._dst_penalties = {}
+        self._src_penalties = {}
+
+        self._print_params()
+
+    def _print_params(self):
+        logging.debug('Parameter Overview:')
+        logging.debug('Algorithm: t2_free_space_only_pop_with_network')
+        logging.debug('Datatypes: %s' % ','.join(self._datatypes))
+        logging.debug('Max bytes/files per hour: %d / %d' % (self._max_bytes_hour, self._max_files_hour))
+        logging.debug('Max bytes/files per hour per RSE: %d / %d' % (self._max_bytes_hour_rse, self._max_files_hour_rse))
+        logging.debug('Min recent requests / Min popularity: %d / %d' % (self._min_recent_requests, self._min_popularity))
+        logging.debug('Max existing replicas: %d' % self._max_replicas)
+
+    def __update_penalties(self):
+        for rse_id, penalty in self._dst_penalties.items():
+            if penalty < 100.0:
+                self._dst_penalties[rse_id] += 10.0
+
+        for rse_id, penalty in self._src_penalties.items():
+            if penalty < 100.0:
+                self._src_penalties[rse_id] += 10.0
+
+    def check_did(self, did):
+        decision = {'did': '{}:{}'.format(did[0].internal, did[1])}
+        if (self._added_cache.check_dataset(decision['did'])):
+            decision['error_reason'] = 'already added replica for this did in the last 24h'
+            return decision
+
+        if (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
+            decision['error_reason'] = 'not a data or mc dataset'
+            return decision
+
+        datatype = did[1].split('.')[4].split('_')[0]
+
+        if datatype not in self._datatypes:
+            decision['error_reason'] = 'wrong datatype'
+            return decision
+
+        try:
+            meta = get_did(did[0], did[1])
+        except DataIdentifierNotFound:
+            decision['error_reason'] = 'did does not exist'
+            return decision
+        if meta['length'] is None:
+            meta['length'] = 0
+        if meta['bytes'] is None:
+            meta['bytes'] = 0
+        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
+
+        decision['length'] = meta['length']
+        decision['bytes'] = meta['bytes']
+
+        total_added_bytes = sum(self._added_bytes.get_series('total'))
+        total_added_files = sum(self._added_files.get_series('total'))
+
+        logging.debug("total_added_bytes: %d" % total_added_bytes)
+        logging.debug("total_added_files: %d" % total_added_files)
+
+        if ((total_added_bytes + meta['bytes']) > self._max_bytes_hour):
+            decision['error_reason'] = 'above bytes limit of %d bytes' % self._max_bytes_hour
+            return decision
+        if ((total_added_files + meta['length']) > self._max_files_hour):
+            decision['error_reason'] = 'above files limit of %d files' % self._max_files_hour
+            return decision
+
+        last_accesses = self._dc.get_did(did)
+        self._dc.add_did(did)
+
+        decision['last_accesses'] = last_accesses
+
+        try:
+            pop = get_popularity(did)
+            decision['popularity'] = pop or 0.0
+        except Exception:
+            decision['error_reason'] = 'problems connecting to ES'
+            return decision
+
+        if (last_accesses < self._min_recent_requests) and (pop < self._min_popularity):
+            decision['error_reason'] = 'did not popular enough'
+            return decision
+
+        return decision
+
+    def place(self, did):
+        self.__update_penalties()
+        self._added_bytes.trim()
+        self._added_files.trim()
+
+        decision = self.check_did(did)
+
+        if 'error_reason' in decision:
+            return decision
+
+        meta = get_did(did[0], did[1])
+        available_reps = {}
+        reps = list_dataset_replicas(did[0], did[1])
+        num_reps = 0
+        space_info = self._fsc.get_rse_space()
+        max_mbps = 0.0
+        for rep in reps:
+            rse_attr = list_rse_attributes(rep['rse_id'])
+            src_rse_id = rep['rse_id']
+            if 'site' not in rse_attr:
+                continue
+
+            src_site = rse_attr['site']
+            src_rse_info = get_rse(rse_id=src_rse_id)
+
+            if 'type' not in rse_attr:
+                continue
+            if rse_attr['type'] != 'DATADISK':
+                continue
+            if not src_rse_info['availability_read']:
+                continue
+
+            if rep['state'] == ReplicaState.AVAILABLE:
+                if rep['available_length'] == 0:
+                    continue
+                net_metrics = {}
+                net_metrics_type = None
+                for metric_type in ('fts', 'fax', 'perfsonar', 'dashb'):
+                    net_metrics_type = metric_type
+                    net_metrics = self._nmc.getMbps(src_site, metric_type)
+                    if net_metrics:
+                        break
+                if len(net_metrics) == 0:
+                    continue
+                available_reps[src_rse_id] = {}
+                for dst_site, mbps in net_metrics.items():
+                    if src_site == dst_site:
+                        continue
+                    if dst_site in self._sites:
+                        if mbps > max_mbps:
+                            max_mbps = mbps
+                        dst_rse_id = self._sites[dst_site]['rse_id']
+                        dst_rse_info = get_rse(rse_id=dst_rse_id)
+
+                        if not dst_rse_info['availability_write']:
+                            continue
+
+                        site_added_bytes = sum(self._added_bytes.get_series(dst_rse_id))
+                        site_added_files = sum(self._added_files.get_series(dst_rse_id))
+
+                        if ((site_added_bytes + meta['bytes']) > self._max_bytes_hour_rse):
+                            continue
+                        if ((site_added_files + meta['length']) > self._max_files_hour_rse):
+                            continue
+
+                        queued = self._nmc.getQueuedFiles(src_site, dst_site)
+
+                        # logging.debug('queued %s -> %s: %d' % (src_site, dst_site, queued))
+                        if queued > 0:
+                            continue
+                        rse_space = space_info.get(dst_rse_id, {'free': 0, 'total': 1})
+                        if src_rse_id not in self._src_penalties:
+                            self._src_penalties[src_rse_id] = 100.0
+                        src_penalty = self._src_penalties[src_rse_id]
+                        if dst_rse_id not in self._dst_penalties:
+                            self._dst_penalties[dst_rse_id] = 100.0
+                        dst_penalty = self._dst_penalties[dst_rse_id]
+
+                        free_space = float(rse_space['free']) / float(rse_space['total']) * 100.0
+                        available_reps[src_rse_id][dst_rse_id] = {'free_space': free_space, 'src_penalty': src_penalty, 'dst_penalty': dst_penalty, 'mbps': float(mbps), 'metrics_type': net_metrics_type}
+
+                num_reps += 1
+
+        # decision['replica_rses'] = available_reps
+        decision['num_replicas'] = num_reps
+
+        if num_reps >= 5:
+            decision['error_reason'] = 'more than 4 replicas already exist'
+            return decision
+
+        src_dst_ratios = []
+
+        if max_mbps == 0.0:
+            decision['error_reason'] = 'could not find enough network metrics'
+            return decision
+
+        for src_id, dst_ids in available_reps.items():
+            for dst_id, metrics in dst_ids.items():
+                if dst_id in available_reps:
+                    continue
+                bdw = (metrics['mbps'] / max_mbps) * 100.0
+                src_penalty = self._src_penalties[src_id]
+                dst_penalty = self._dst_penalties[dst_id]
+
+                ratio = ((metrics['free_space'] / 4.0) + bdw) * src_penalty * dst_penalty
+                src_dst_ratios.append((src_id, dst_id, ratio))
+
+        if len(src_dst_ratios) == 0:
+            decision['error_reason'] = 'found no suitable src/dst for replication'
+            return decision
+
+        sorted_ratios = sorted(src_dst_ratios, key=itemgetter(2), reverse=True)
+        logging.debug(sorted_ratios)
+        destination_rse = sorted_ratios[0][1]
+        source_rse = sorted_ratios[0][0]
+        decision['destination_rse'] = get_rse_name(rse_id=destination_rse)
+        decision['source_rse'] = get_rse_name(rse_id=source_rse)
+        # decision['rse_ratios'] = src_dst_ratios
+        self._dst_penalties[destination_rse] = 10.0
+        self._src_penalties[source_rse] = 10.0
+
+        self._added_cache.add_dataset(decision['did'])
+
+        self._added_bytes.add_point(destination_rse, meta['bytes'])
+        self._added_files.add_point(destination_rse, meta['length'])
+
+        self._added_bytes.add_point('total', meta['bytes'])
+        self._added_files.add_point('total', meta['length'])
+
+        return decision