rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py

@@ -1,294 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from operator import itemgetter
-from typing import TYPE_CHECKING, Any
-
-from rucio.common.config import config_get, config_get_int
-from rucio.common.constants import RseAttr
-from rucio.common.exception import DataIdentifierNotFound
-from rucio.core.did import get_did
-from rucio.core.replica import list_dataset_replicas
-from rucio.core.rse import get_rse, get_rse_name, list_rse_attributes
-from rucio.core.rse_expression_parser import parse_expression
-from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
-from rucio.daemons.c3po.collectors.network_metrics import NetworkMetricsCollector
-from rucio.daemons.c3po.utils.dataset_cache import DatasetCache
-from rucio.daemons.c3po.utils.expiring_dataset_cache import ExpiringDatasetCache
-from rucio.daemons.c3po.utils.popularity import get_popularity
-from rucio.daemons.c3po.utils.timeseries import RedisTimeSeries
-from rucio.db.sqla.constants import ReplicaState
-
-if TYPE_CHECKING:
-    from rucio.common.types import InternalScope
-
-
-class PlacementAlgorithm:
-    """
-    Placement algorithm that focusses on free space on T2 DATADISK RSEs. It incorporates network metrics for placement decisions.
-    """
-    def __init__(
-            self,
-            datatypes: str,
-            dest_rse_expr: str,
-            max_bytes_hour: int,
-            max_files_hour: int,
-            max_bytes_hour_rse: int,
-            max_files_hour_rse: int,
-            min_popularity: int,
-            min_recent_requests: int,
-            max_replicas: int
-    ):
-        self._fsc = FreeSpaceCollector()
-        self._nmc = NetworkMetricsCollector()
-        self._added_cache = ExpiringDatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
-        self._dc = DatasetCache(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), timeout=86400)
-        self._added_bytes = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_bytes_")
-        self._added_files = RedisTimeSeries(config_get('c3po', 'redis_host'), config_get_int('c3po', 'redis_port'), window=3600, prefix="added_files_")
-
-        self._datatypes = datatypes.split(',')
-        self._dest_rse_expr = dest_rse_expr
-        self._max_bytes_hour = max_bytes_hour
-        self._max_files_hour = max_files_hour
-        self._max_bytes_hour_rse = max_bytes_hour_rse
-        self._max_files_hour_rse = max_files_hour_rse
-        self._min_popularity = min_popularity
-        self._min_recent_requests = min_recent_requests
-        self._max_replicas = max_replicas
-
-        rses = parse_expression(self._dest_rse_expr)
-
-        self._rses = {}
-        self._sites = {}
-        for rse in rses:
-            rse_attrs = list_rse_attributes(rse_id=rse['id'])
-            rse_attrs['rse_id'] = rse['id']
-            self._rses[rse['id']] = rse_attrs
-            self._sites[rse_attrs[RseAttr.SITE]] = rse_attrs
-
-        self._dst_penalties = {}
-        self._src_penalties = {}
-
-        self._print_params()
-
-    def _print_params(self) -> None:
-        logging.debug('Parameter Overview:')
-        logging.debug('Algorithm: t2_free_space_only_pop_with_network')
-        logging.debug('Datatypes: %s' % ','.join(self._datatypes))
-        logging.debug('Max bytes/files per hour: %d / %d' % (self._max_bytes_hour, self._max_files_hour))
-        logging.debug('Max bytes/files per hour per RSE: %d / %d' % (self._max_bytes_hour_rse, self._max_files_hour_rse))
-        logging.debug('Min recent requests / Min popularity: %d / %d' % (self._min_recent_requests, self._min_popularity))
-        logging.debug('Max existing replicas: %d' % self._max_replicas)
-
-    def __update_penalties(self) -> None:
-        for rse_id, penalty in self._dst_penalties.items():
-            if penalty < 100.0:
-                self._dst_penalties[rse_id] += 10.0
-
-        for rse_id, penalty in self._src_penalties.items():
-            if penalty < 100.0:
-                self._src_penalties[rse_id] += 10.0
-
-    def check_did(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
-        decision: dict[str, Any] = {'did': '{}:{}'.format(did[0].internal, did[1])}
-        if (self._added_cache.check_dataset(decision['did'])):
-            decision['error_reason'] = 'already added replica for this did in the last 24h'
-            return decision
-
-        if (did[0].external is not None) and (not did[0].external.startswith('data')) and (not did[0].external.startswith('mc')):
-            decision['error_reason'] = 'not a data or mc dataset'
-            return decision
-
-        datatype = did[1].split('.')[4].split('_')[0]
-
-        if datatype not in self._datatypes:
-            decision['error_reason'] = 'wrong datatype'
-            return decision
-
-        try:
-            meta = get_did(did[0], did[1])
-        except DataIdentifierNotFound:
-            decision['error_reason'] = 'did does not exist'
-            return decision
-        if meta['length'] is None:
-            meta['length'] = 0
-        if meta['bytes'] is None:
-            meta['bytes'] = 0
-        logging.debug('got %s:%s, num_files: %d, bytes: %d' % (did[0], did[1], meta['length'], meta['bytes']))
-
-        decision['length'] = meta['length']
-        decision['bytes'] = meta['bytes']
-
-        total_added_bytes = sum(self._added_bytes.get_series('total'))
-        total_added_files = sum(self._added_files.get_series('total'))
-
-        logging.debug("total_added_bytes: %d" % total_added_bytes)
-        logging.debug("total_added_files: %d" % total_added_files)
-
-        if ((total_added_bytes + meta['bytes']) > self._max_bytes_hour):
-            decision['error_reason'] = 'above bytes limit of %d bytes' % self._max_bytes_hour
-            return decision
-        if ((total_added_files + meta['length']) > self._max_files_hour):
-            decision['error_reason'] = 'above files limit of %d files' % self._max_files_hour
-            return decision
-
-        last_accesses = self._dc.get_did(did)
-        self._dc.add_did(did)
-
-        decision['last_accesses'] = last_accesses
-
-        try:
-            pop = get_popularity(did)
-            decision['popularity'] = pop or 0.0
-        except Exception:
-            decision['error_reason'] = 'problems connecting to ES'
-            return decision
-
-        if (last_accesses < self._min_recent_requests) and (decision['popularity'] < self._min_popularity):
-            decision['error_reason'] = 'did not popular enough'
-            return decision
-
-        return decision
-
-    def place(self, did: tuple['InternalScope', str]) -> dict[str, Any]:
-        self.__update_penalties()
-        self._added_bytes.trim()
-        self._added_files.trim()
-
-        decision = self.check_did(did)
-
-        if 'error_reason' in decision:
-            return decision
-
-        meta = get_did(did[0], did[1])
-        available_reps = {}
-        reps = list_dataset_replicas(did[0], did[1])
-        num_reps = 0
-        space_info = self._fsc.get_rse_space()
-        max_mbps = 0.0
-        for rep in reps:
-            rse_attr = list_rse_attributes(rep['rse_id'])
-            src_rse_id = rep['rse_id']
-            if RseAttr.SITE not in rse_attr:
-                continue
-
-            src_site = rse_attr[RseAttr.SITE]
-            src_rse_info = get_rse(rse_id=src_rse_id)
-
-            if 'type' not in rse_attr:
-                continue
-            if rse_attr['type'] != 'DATADISK':
-                continue
-            if not src_rse_info['availability_read']:
-                continue
-
-            if rep['state'] == ReplicaState.AVAILABLE:
-                if rep['available_length'] == 0:
-                    continue
-                net_metrics = {}
-                net_metrics_type = None
-                for metric_type in ('fts', 'fax', 'perfsonar', 'dashb'):
-                    net_metrics_type = metric_type
-                    net_metrics = self._nmc.getMbps(src_site, metric_type)
-                    if net_metrics:
-                        break
-                if len(net_metrics) == 0:  # type: ignore
-                    continue
-                available_reps[src_rse_id] = {}
-                for dst_site, mbps in net_metrics.items():  # type: ignore
-                    if src_site == dst_site:
-                        continue
-                    if dst_site in self._sites:
-                        if mbps > max_mbps:
-                            max_mbps = mbps
-                        dst_rse_id = self._sites[dst_site]['rse_id']
-                        dst_rse_info = get_rse(rse_id=dst_rse_id)
-
-                        if not dst_rse_info['availability_write']:
-                            continue
-
-                        site_added_bytes = sum(self._added_bytes.get_series(dst_rse_id))
-                        site_added_files = sum(self._added_files.get_series(dst_rse_id))
-
-                        if ((site_added_bytes + meta['bytes']) > self._max_bytes_hour_rse):
-                            continue
-                        if ((site_added_files + meta['length']) > self._max_files_hour_rse):
-                            continue
-
-                        queued = self._nmc.getQueuedFiles(src_site, dst_site)
-
-                        # logging.debug('queued %s -> %s: %d' % (src_site, dst_site, queued))
-                        if queued > 0:
-                            continue
-                        rse_space = space_info.get(dst_rse_id, {'free': 0, 'total': 1})
-                        if src_rse_id not in self._src_penalties:
-                            self._src_penalties[src_rse_id] = 100.0
-                        src_penalty = self._src_penalties[src_rse_id]
-                        if dst_rse_id not in self._dst_penalties:
-                            self._dst_penalties[dst_rse_id] = 100.0
-                        dst_penalty = self._dst_penalties[dst_rse_id]
-
-                        free_space = float(rse_space['free']) / float(rse_space['total']) * 100.0
-                        available_reps[src_rse_id][dst_rse_id] = {'free_space': free_space, 'src_penalty': src_penalty, 'dst_penalty': dst_penalty, 'mbps': float(mbps), 'metrics_type': net_metrics_type}
-
-                num_reps += 1
-
-        # decision['replica_rses'] = available_reps
-        decision['num_replicas'] = num_reps
-
-        if num_reps >= 5:
-            decision['error_reason'] = 'more than 4 replicas already exist'
-            return decision
-
-        src_dst_ratios = []
-
-        if max_mbps == 0.0:
-            decision['error_reason'] = 'could not find enough network metrics'
-            return decision
-
-        for src_id, dst_ids in available_reps.items():
-            for dst_id, metrics in dst_ids.items():
-                if dst_id in available_reps:
-                    continue
-                bdw = (metrics['mbps'] / max_mbps) * 100.0
-                src_penalty = self._src_penalties[src_id]
-                dst_penalty = self._dst_penalties[dst_id]
-
-                ratio = ((metrics['free_space'] / 4.0) + bdw) * src_penalty * dst_penalty
-                src_dst_ratios.append((src_id, dst_id, ratio))
-
-        if len(src_dst_ratios) == 0:
-            decision['error_reason'] = 'found no suitable src/dst for replication'
-            return decision
-
-        sorted_ratios = sorted(src_dst_ratios, key=itemgetter(2), reverse=True)
-        logging.debug(sorted_ratios)
-        destination_rse = sorted_ratios[0][1]
-        source_rse = sorted_ratios[0][0]
-        decision['destination_rse'] = get_rse_name(rse_id=destination_rse)
-        decision['source_rse'] = get_rse_name(rse_id=source_rse)
-        # decision['rse_ratios'] = src_dst_ratios
-        self._dst_penalties[destination_rse] = 10.0
-        self._src_penalties[source_rse] = 10.0
-
-        self._added_cache.add_dataset(decision['did'])
-
-        self._added_bytes.add_point(destination_rse, meta['bytes'])
-        self._added_files.add_point(destination_rse, meta['length'])
-
-        self._added_bytes.add_point('total', meta['bytes'])
-        self._added_files.add_point('total', meta['length'])
-
-        return decision

rucio/daemons/c3po/c3po.py

@@ -1,371 +0,0 @@
-# Copyright European Organization for Nuclear Research (CERN) since 2012
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-'''
-Dynamic data placement daemon.
-'''
-
-import logging
-from datetime import datetime
-from hashlib import md5
-from json import dumps
-from queue import Queue
-from threading import Event, Thread
-from time import sleep
-from typing import TYPE_CHECKING, Optional
-from uuid import uuid4
-
-from requests import post
-from requests.auth import HTTPBasicAuth
-from requests.exceptions import RequestException
-
-import rucio.db.sqla.util
-from rucio.client import Client
-from rucio.common import exception
-from rucio.common.config import config_get, config_get_options
-from rucio.common.logging import setup_logging
-from rucio.common.types import InternalScope
-from rucio.daemons.c3po.collectors.free_space import FreeSpaceCollector
-from rucio.daemons.c3po.collectors.jedi_did import JediDIDCollector
-from rucio.daemons.c3po.collectors.workload import WorkloadCollector
-
-if TYPE_CHECKING:
-    from types import FrameType
-
-GRACEFUL_STOP = Event()
-DAEMON_NAME = 'c3po'
-
-
-def read_free_space(
-        once: bool = False,
-        thread: int = 0,
-        waiting_time: int = 1800,
-        sleep_time: int = 10
-) -> None:
-    """
-    Thread to collect the space usage information for RSEs.
-    """
-    free_space_collector = FreeSpaceCollector()
-    timer = waiting_time
-    while not GRACEFUL_STOP.is_set():
-        if timer < waiting_time:
-            timer += sleep_time
-            sleep(sleep_time)
-            continue
-
-        logging.info('collecting free space')
-        free_space_collector.collect_free_space()
-        timer = 0
-
-
-def read_workload(
-        once: bool = False,
-        thread: int = 0,
-        waiting_time: int = 1800,
-        sleep_time: int = 10
-) -> None:
-    """
-    Thread to collect the workload information from PanDA.
-    """
-    workload_collector = WorkloadCollector()
-    timer = waiting_time
-    while not GRACEFUL_STOP.is_set():
-        if timer < waiting_time:
-            timer += sleep_time
-            sleep(sleep_time)
-            continue
-
-        logging.info('collecting workload')
-        workload_collector.collect_workload()
-        timer = 0
-
-
-def print_workload(
-        once: bool = False,
-        thread: int = 0,
-        waiting_time: int = 600,
-        sleep_time: int = 10
-) -> None:
-    """
-    Thread to regularly output the workload to logs for debugging.
-    """
-    workload_collector = WorkloadCollector()
-    timer = waiting_time
-    while not GRACEFUL_STOP.is_set():
-        if timer < waiting_time:
-            timer += sleep_time
-            sleep(sleep_time)
-            continue
-
-        logging.info('Number of sites cached %d' % len(workload_collector.get_sites()))
-        for site in workload_collector.get_sites():
-            logging.info('%s: %d / %d / %d' % (site, workload_collector.get_cur_jobs(site), workload_collector.get_avg_jobs(site), workload_collector.get_max_jobs(site)))
-        timer = 0
-
-
-def read_dids(
-        once: bool = False,
-        thread: int = 0,
-        did_collector: Optional[JediDIDCollector] = None,
-        waiting_time: int = 60,
-        sleep_time: int = 10
-) -> None:
-    """
-    Thread to collect DIDs for the placement algorithm.
-    """
-    timer = waiting_time
-    while not GRACEFUL_STOP.is_set():
-        if timer < waiting_time:
-            timer += sleep_time
-            sleep(sleep_time)
-            continue
-
-        if did_collector is not None:
-            did_collector.get_dids()
-        timer = 0
-
-
-def add_rule(
-        client: Client,
-        did: dict[str, str],
-        src_rse: str,
-        dst_rse: str
-) -> None:
-    logging.debug('add rule for %s from %s to %s' % (did, src_rse, dst_rse))
-    r = client.add_replication_rule([did, ], 1, dst_rse, lifetime=604800, account='c3po', source_replica_expression=src_rse, activity='Data Brokering', asynchronous=True)
-    logging.debug(r)
-
-
-def place_replica(
-        did_queue: Queue,
-        once: bool = False,
-        thread: int = 0,
-        waiting_time: int = 100,
-        dry_run: bool = False,
-        sampling: bool = False,
-        algorithms: str = 't2_free_space_only_pop_with_network',
-        datatypes: str = 'NTUP,DAOD',
-        dest_rse_expr: str = 'type=DATADISK',
-        max_bytes_hour: int = 100000000000000,
-        max_files_hour: int = 100000,
-        max_bytes_hour_rse: int = 50000000000000,
-        max_files_hour_rse: int = 10000,
-        min_popularity: int = 8,
-        min_recent_requests: int = 5,
-        max_replicas: int = 5,
-        sleep_time: int = 10
-) -> None:
-    """
-    Thread to run the placement algorithm to decide if and where to put new replicas.
-    """
-    try:
-        c3po_options = config_get_options('c3po')
-        client = None
-
-        if 'algorithms' in c3po_options:
-            algorithms = config_get('c3po', 'algorithms')
-
-        algorithms_list = algorithms.split(',')
-
-        if not dry_run:
-            if len(algorithms_list) != 1:
-                logging.error('Multiple algorithms are only allowed in dry_run mode')
-                return
-            client = Client(auth_type='x509_proxy', account='c3po', creds={'client_proxy': '/opt/rucio/etc/ddmadmin.long.proxy'})
-
-        vo = client.vo
-        instances = {}
-        for algorithm in algorithms_list:
-            module_path = 'rucio.daemons.c3po.algorithms.' + algorithm
-            module = __import__(module_path, globals(), locals(), ['PlacementAlgorithm'])
-            instance = module.PlacementAlgorithm(datatypes, dest_rse_expr, max_bytes_hour, max_files_hour, max_bytes_hour_rse, max_files_hour_rse, min_popularity, min_recent_requests, max_replicas)
-            instances[algorithm] = instance
-
-        params = {
-            'dry_run': dry_run,
-            'sampling': sampling,
-            'datatypes': datatypes,
-            'dest_rse_expr': dest_rse_expr,
-            'max_bytes_hour': max_bytes_hour,
-            'max_files_hour': max_files_hour,
-            'max_bytes_hour_rse': max_bytes_hour_rse,
-            'max_files_hour_rse': max_files_hour_rse,
-            'min_recent_requests': min_recent_requests,
-            'min_popularity': min_popularity
-        }
-
-        instance_id = str(uuid4()).split('-')[0]
-
-        elastic_url = config_get('c3po', 'elastic_url')
-        elastic_index = config_get('c3po', 'elastic_index')
-
-        ca_cert = False
-        if 'ca_cert' in c3po_options:
-            ca_cert = config_get('c3po', 'ca_cert')
-
-        auth = False
-        if ('elastic_user' in c3po_options) and ('elastic_pass' in c3po_options):
-            auth = HTTPBasicAuth(config_get('c3po', 'elastic_user'), config_get('c3po', 'elastic_pass'))
-
-        w = waiting_time
-        while not GRACEFUL_STOP.is_set():
-            if w < waiting_time:
-                w += sleep_time
-                sleep(sleep_time)
-                continue
-            len_dids = did_queue.qsize()
-
-            if len_dids > 0:
-                logging.debug('(%s) %d did(s) in queue' % (instance_id, len_dids))
-            else:
-                logging.debug('(%s) no dids in queue' % (instance_id))
-
-            for _ in range(0, len_dids):
-                did = did_queue.get()
-                if isinstance(did[0], str):
-                    did[0] = InternalScope(did[0], vo=vo)
-                for algorithm, instance in instances.items():
-                    logging.info('(%s:%s) Retrieved %s:%s from queue. Run placement algorithm' % (algorithm, instance_id, did[0], did[1]))
-                    decision = instance.place(did)
-                    decision['@timestamp'] = datetime.utcnow().isoformat()
-                    decision['algorithm'] = algorithm
-                    decision['instance_id'] = instance_id
-                    decision['params'] = params
-
-                    create_rule = True
-                    if sampling and 'error_reason' not in decision:
-                        create_rule = bool(ord(md5(decision['did']).hexdigest()[-1]) & 1)
-                        decision['create_rule'] = create_rule
-                    # write the output to ES for further analysis
-                    index_url = elastic_url + '/' + elastic_index + '-' + datetime.utcnow().strftime('%Y-%m') + '/record/'
-                    try:
-                        if ca_cert:
-                            r = post(index_url, data=dumps(decision), verify=ca_cert, auth=auth)
-                        else:
-                            r = post(index_url, data=dumps(decision))
-                        if r.status_code != 201:
-                            logging.error(r)
-                            logging.error('(%s:%s) could not write to ElasticSearch' % (algorithm, instance_id))
-                    except RequestException as e:
-                        logging.error('(%s:%s) could not write to ElasticSearch' % (algorithm, instance_id))
-                        logging.error(e)
-                        continue
-
-                    logging.debug(decision)
-                    if 'error_reason' in decision:
-                        logging.error('(%s:%s) The placement algorithm ran into an error: %s' % (algorithm, instance_id, decision['error_reason']))
-                        continue
-
-                    logging.info('(%s:%s) Decided to place a new replica for %s on %s' % (algorithm, instance_id, decision['did'], decision['destination_rse']))
-
-                    if (not dry_run) and create_rule:
-                        # DO IT!
-                        try:
-                            add_rule(client, {'scope': did[0].external, 'name': did[1]}, decision.get('source_rse'), decision.get('destination_rse'))  # type: ignore
-                        except exception.RucioException as e:
-                            logging.debug(e)
-
-            w = 0
-    except Exception as e:
-        logging.critical(e)
-
-
-def stop(signum: Optional[int] = None, frame: Optional['FrameType'] = None) -> None:
-    """
-    Graceful exit.
-    """
-    GRACEFUL_STOP.set()
-
-
-def run(
-        once: bool = False,
-        threads: int = 1,
-        only_workload: bool = False,
-        dry_run: bool = False,
-        sampling: bool = False,
-        algorithms: str = 't2_free_space_only_pop_with_network',
-        datatypes: str = 'NTUP,DAOD',
-        dest_rse_expr: str = 'type=DATADISK',
-        max_bytes_hour: int = 100000000000000,
-        max_files_hour: int = 100000,
-        max_bytes_hour_rse: int = 50000000000000,
-        max_files_hour_rse: int = 10000,
-        min_popularity: int = 8,
-        min_recent_requests: int = 5,
-        max_replicas: int = 5,
-        waiting_time_read_free_space: int = 1800,
-        waiting_time_read_workload: int = 1800,
-        waiting_time_print_workload: int = 600,
-        waiting_time_read_dids: int = 60,
-        waiting_time_place_replica: int = 100,
-        sleep_time: int = 10
-) -> None:
-    """
-    Starts up the main thread
-    """
-    setup_logging(process_name=DAEMON_NAME)
-
-    if rucio.db.sqla.util.is_old_db():
-        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
-
-    logging.info('activating C-3PO')
-
-    thread_list = []
-    try:
-        if only_workload:
-            logging.info('running in workload-collector-only mode')
-            thread_list.append(Thread(target=read_workload, name='read_workload', kwargs={'thread': 0,
-                                                                                          'waiting_time': waiting_time_read_workload,
-                                                                                          'sleep_time': sleep_time}))
-            thread_list.append(Thread(target=print_workload, name='print_workload', kwargs={'thread': 0,
-                                                                                            'waiting_time': waiting_time_print_workload,
-                                                                                            'sleep_time': sleep_time}))
-        else:
-            logging.info('running in placement mode')
-            did_queue = Queue()
-            dc = JediDIDCollector(did_queue)
-
-            thread_list.append(Thread(target=read_free_space, name='read_free_space', kwargs={'thread': 0,
-                                                                                              'waiting_time': waiting_time_read_free_space,
-                                                                                              'sleep_time': sleep_time}))
-            thread_list.append(Thread(target=read_dids, name='read_dids', kwargs={'thread': 0,
-                                                                                  'did_collector': dc,
-                                                                                  'waiting_time': waiting_time_read_dids,
-                                                                                  'sleep_time': sleep_time}))
-            thread_list.append(Thread(target=place_replica, name='place_replica', kwargs={'thread': 0,
-                                                                                          'did_queue': did_queue,
-                                                                                          'waiting_time': waiting_time_place_replica,
-                                                                                          'algorithms': algorithms,
-                                                                                          'dry_run': dry_run,
-                                                                                          'sampling': sampling,
-                                                                                          'datatypes': datatypes,
-                                                                                          'dest_rse_expr': dest_rse_expr,
-                                                                                          'max_bytes_hour': max_bytes_hour,
-                                                                                          'max_files_hour': max_files_hour,
-                                                                                          'max_bytes_hour_rse': max_bytes_hour_rse,
-                                                                                          'max_files_hour_rse': max_files_hour_rse,
-                                                                                          'min_popularity': min_popularity,
-                                                                                          'min_recent_requests': min_recent_requests,
-                                                                                          'max_replicas': max_replicas,
-                                                                                          'sleep_time': sleep_time}))
-
-        for t in thread_list:
-            t.start()
-
-        logging.info('waiting for interrupts')
-
-        while len(thread_list) > 0:
-            [t.join(timeout=3) for t in thread_list if t and t.is_alive()]
-    except Exception as error:
-        logging.critical(error)