rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/c3po/algorithms/__init__.py

@@ -1,13 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

rucio/daemons/c3po/algorithms/simple.py

@@ -1,134 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from operator import itemgetter
-from random import shuffle
-from typing import TYPE_CHECKING, Any
-
-from rucio.common.exception import DataIdentifierNotFound
-from rucio.core.did import get_did
-from rucio.core.replica import list_dataset_replicas
-from rucio.daemons.c3po.collectors.agis import MappingCollector
-from rucio.daemons.c3po.collectors.workload import WorkloadCollector
-from rucio.db.sqla.constants import ReplicaState
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-
-class PlacementAlgorithm:
-    def __init__(self):
-        self._mc = MappingCollector()
-        self._wc = WorkloadCollector()
-        self.__setup_penalties()
-
-    def __setup_penalties(self) -> None:
-        self._penalties = {}
-        for panda_site in self._wc.get_sites():
-            site = self._mc.panda_to_site(panda_site)
-            self._penalties[site] = 0.1
-
-    def __update_penalties(self) -> None:
-        for site, penalty in self._penalties.items():
-            if penalty > 0.1:
-                self._penalties[site] = penalty - 0.1
-
-    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
-        self.__update_penalties()
-        decision: dict[str, Any] = {'did': '{}:{}'.format(did['scope'].internal, did['name'])}  # type: ignore (did is treated as a dict here, and as a tuple everywhere else)
-        try:
-            meta = get_did(did[0], did[1])
-        except DataIdentifierNotFound:
-            decision['error_reason'] = 'did does not exist'
-            return decision
-        if meta['length'] is None:
-            meta['length'] = 0
-        if meta['bytes'] is None:
-            meta['bytes'] = 0
-        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
-
-        decision['length'] = meta['length']
-        decision['bytes'] = meta['bytes']
-
-        available_rses = []
-        available_sites = []
-        reps = list_dataset_replicas(did[0], did[1])
-
-        num_reps = 0
-        for rep in reps:
-            if rep['state'] == ReplicaState.AVAILABLE:
-                available_rses.append(rep['rse'])
-                available_sites.append(self._mc.ddm_to_site(rep['rse']))
-                num_reps += 1
-
-        decision['replica_rses'] = available_rses
-        decision['num_replicas'] = num_reps
-        if num_reps >= 5:
-            decision['error_reason'] = 'more than 4 replicas already exist'
-            return decision
-
-        site_ratios = {}
-        site_job_info = {}
-        for panda_site in self._wc.get_sites():
-            site = self._mc.panda_to_site(panda_site)
-            job_info = self._wc.get_job_info(panda_site)
-            ratio = float(job_info[0]) / (float(job_info[1]) + float(job_info[2]) / 2)
-            penalty = self._penalties[site]
-            site_ratios[site] = ratio * penalty
-            site_job_info[site] = (job_info, penalty)
-
-        decision['site_ratios'] = site_ratios
-        decision['site_job_info'] = site_job_info
-        picked_site = None
-        picked_rse = None
-
-        for site, _ in sorted(site_ratios.items(), key=itemgetter(1)):
-            if site in available_sites:
-                continue
-            rses_for_site = self._mc.site_to_ddm(site)
-            if rses_for_site is None:
-                continue
-
-            for rse in rses_for_site:
-                if 'DATADISK' in rse:
-                    picked_rse = rse
-                    picked_site = site
-                    break
-            if picked_rse:
-                break
-
-        if picked_rse is None:
-            decision['error_reason'] = 'could not pick RSE'
-            return decision
-
-        decision['destination_rse'] = picked_rse
-        if picked_site:
-            self._penalties[site] = 1
-
-        picked_source = None
-        shuffle(available_rses)
-        for rse in available_rses:
-            if 'TAPE' in rse:
-                continue
-            picked_source = rse
-            break
-
-        if picked_source is None:
-            picked_source = available_rses[0]
-
-        decision['source_rse'] = picked_source
-        logging.debug("Picked %s as source and %s as destination RSE" % (picked_source, picked_rse))
-
-        return decision

rucio/daemons/c3po/algorithms/t2_free_space.py

@@ -1,128 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from operator import itemgetter
-from typing import TYPE_CHECKING, Any
-
-from rucio.common.config import config_get, config_get_int
-from rucio.common.exception import DataIdentifierNotFound
-from rucio.core.did import get_did
-from rucio.core.replica import list_dataset_replicas
-from rucio.core.rse import get_rse_name, list_rse_attributes
-from rucio.core.rse_expression_parser import parse_expression
-from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
-from rucio.daemons.c3po.utils.dataset_cache import DatasetCache
-from rucio.daemons.c3po.utils.popularity import get_popularity
-from rucio.db.sqla.constants import ReplicaState
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-
-class PlacementAlgorithm:
-    """
-    Placement algorithm that focusses on free space on T2 DATADISK RSEs.
-    """
-    def __init__(self):
-        self._fsc = FreeSpaceCollector()
-        self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
-
-        rse_expr = "tier=2&type=DATADISK"
-        rse_attrs = parse_expression(rse_expr)
-
-        self._rses = []
-        for rse in rse_attrs:
-            self._rses.append(rse['id'])
-
-        self.__setup_penalties()
-
-    def __setup_penalties(self) -> None:
-        self._penalties = {}
-        for rse_id in self._rses:
-            self._penalties[rse_id] = 1.0
-
-    def __update_penalties(self) -> None:
-        for rse_id, penalty in self._penalties.items():
-            if penalty > 1.0:
-                self._penalties[rse_id] = penalty - 1
-
-    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
-        self.__update_penalties()
-        decision: dict[str, Any] = {'did': '{}:{}'.format(did[0].internal, did[1])}
-        if (did[0].external is not None) and (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
-            decision['error_reason'] = 'not a data or mc dataset'
-            return decision
-
-        try:
-            meta = get_did(did[0], did[1])
-        except DataIdentifierNotFound:
-            decision['error_reason'] = 'did does not exist'
-            return decision
-        if meta['length'] is None:
-            meta['length'] = 0
-        if meta['bytes'] is None:
-            meta['bytes'] = 0
-        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
-
-        decision['length'] = meta['length']
-        decision['bytes'] = meta['bytes']
-
-        last_accesses = self._dc.get_did(did)
-        self._dc.add_did(did)
-
-        decision['last_accesses'] = last_accesses
-
-        pop = get_popularity(did)
-        decision['popularity'] = pop or 0.0
-
-        if (last_accesses < 5) and (pop < 10.0):
-            decision['error_reason'] = 'did not popular enough'
-            return decision
-
-        free_rses = self._rses
-        available_reps = []
-        reps = list_dataset_replicas(did[0], did[1])
-        num_reps = 0
-        for rep in reps:
-            rse_attr = list_rse_attributes(rep['rse_id'])
-            if 'type' not in rse_attr:
-                continue
-            if rse_attr['type'] != 'DATADISK':
-                continue
-            if rep['state'] == ReplicaState.AVAILABLE:
-                if rep['rse_id'] in free_rses:
-                    free_rses.remove(rep['rse_id'])
-                available_reps.append(rep['rse_id'])
-                num_reps += 1
-
-        decision['replica_rses'] = available_reps
-        decision['num_replicas'] = num_reps
-        if num_reps >= 5:
-            decision['error_reason'] = 'more than 4 replicas already exist'
-            return decision
-
-        rse_ratios = {}
-        space_info = self._fsc.get_rse_space()
-        for rse_id in free_rses:
-            rse_space = space_info[rse_id]
-            penalty = self._penalties[rse_id]
-            rse_ratios[rse_id] = float(rse_space['free']) / float(rse_space['total']) * 100.0 / penalty
-
-        sorted_rses = sorted(rse_ratios.items(), key=itemgetter(1), reverse=True)
-        decision['destination_rse'] = get_rse_name(rse_id=sorted_rses[0][0])
-        decision['rse_ratios'] = sorted_rses
-        self._penalties[sorted_rses[0][0]] = 10.0
-
-        return decision

rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py

@@ -1,130 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from operator import itemgetter
-from typing import TYPE_CHECKING, Any
-
-from rucio.common.config import config_get, config_get_int
-from rucio.common.exception import DataIdentifierNotFound
-from rucio.core.did import get_did
-from rucio.core.replica import list_dataset_replicas
-from rucio.core.rse import get_rse_name, list_rse_attributes
-from rucio.core.rse_expression_parser import parse_expression
-from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
-from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache
-from rucio.daemons.c3po.utils.popularity import get_popularity
-from rucio.db.sqla.constants import ReplicaState
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-
-class PlacementAlgorithm:
-    """
-    Placement algorithm that focusses on free space on T2 DATADISK RSEs.
-    """
-    def __init__(self):
-        self._fsc = FreeSpaceCollector()
-        self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
-
-        rse_expr = "tier=2&type=DATADISK"
-        rse_attrs = parse_expression(rse_expr)
-
-        self._rses = []
-        for rse in rse_attrs:
-            self._rses.append(rse['id'])
-
-        self.__setup_penalties()
-
-    def __setup_penalties(self) -> None:
-        self._penalties = {}
-        for rse_id in self._rses:
-            self._penalties[rse_id] = 1.0
-
-    def __update_penalties(self) -> None:
-        for rse_id, penalty in self._penalties.items():
-            if penalty > 1.0:
-                self._penalties[rse_id] = penalty - 1
-
-    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
-        self.__update_penalties()
-        decision: dict[str, Any] = {'did': '{}:{}'.format(did[0].internal, did[1])}
-        if (self._added_cache.check_dataset(decision['did'])):
-            decision['error_reason'] = 'already added replica for this did in the last 24h'
-            return decision
-
-        if (did[0].external is not None) and (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
-            decision['error_reason'] = 'not a data or mc dataset'
-            return decision
-
-        try:
-            meta = get_did(did[0], did[1])
-        except DataIdentifierNotFound:
-            decision['error_reason'] = 'did does not exist'
-            return decision
-        if meta['length'] is None:
-            meta['length'] = 0
-        if meta['bytes'] is None:
-            meta['bytes'] = 0
-        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
-
-        decision['length'] = meta['length']
-        decision['bytes'] = meta['bytes']
-
-        pop = get_popularity(did)
-        decision['popularity'] = pop or 0.0
-
-        if (pop < 10.0):
-            decision['error_reason'] = 'did not popular enough'
-            return decision
-
-        free_rses = self._rses
-        available_reps = []
-        reps = list_dataset_replicas(did[0], did[1])
-        num_reps = 0
-        for rep in reps:
-            rse_attr = list_rse_attributes(rse_id=rep['rse_id'])
-            if 'type' not in rse_attr:
-                continue
-            if rse_attr['type'] != 'DATADISK':
-                continue
-            if rep['state'] == ReplicaState.AVAILABLE:
-                if rep['rse_id'] in free_rses:
-                    free_rses.remove(rep['rse_id'])
-                available_reps.append(rep['rse_id'])
-                num_reps += 1
-
-        decision['replica_rses'] = available_reps
-        decision['num_replicas'] = num_reps
-
-        if num_reps >= 5:
-            decision['error_reason'] = 'more than 4 replicas already exist'
-            return decision
-
-        rse_ratios = {}
-        space_info = self._fsc.get_rse_space()
-        for rse_id in free_rses:
-            rse_space = space_info[rse_id]
-            penalty = self._penalties[rse_id]
-            rse_ratios[rse_id] = float(rse_space['free']) / float(rse_space['total']) * 100.0 / penalty
-
-        sorted_rses = sorted(rse_ratios.items(), key=itemgetter(1), reverse=True)
-        decision['destination_rse'] = get_rse_name(sorted_rses[0][0])
-        decision['rse_ratios'] = sorted_rses
-        self._penalties[sorted_rses[0][0]] = 10.0
-
-        self._added_cache.add_dataset(decision['did'])
-
-        return decision