rucio 35.7.0__py3-none-any.whl

rucio/core/transfer.py

@@ -0,0 +1,1517 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import logging
+import operator
+import re
+import sys
+import time
+import traceback
+from collections import defaultdict
+from typing import TYPE_CHECKING, cast
+
+from dogpile.cache import make_region
+from dogpile.cache.api import NoValue
+from sqlalchemy import select, update
+from sqlalchemy.exc import IntegrityError
+
+from rucio.common import constants
+from rucio.common.config import config_get, config_get_list
+from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr
+from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation
+from rucio.common.utils import construct_non_deterministic_pfn
+from rucio.core import did
+from rucio.core import message as message_core
+from rucio.core import request as request_core
+from rucio.core.account import list_accounts
+from rucio.core.monitor import MetricManager
+from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state
+from rucio.core.rse import RseData
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection
+from rucio.db.sqla.session import read_session, stream_session, transactional_session
+from rucio.rse import rsemanager as rsemgr
+from rucio.transfertool.bittorrent import BittorrentTransfertool
+from rucio.transfertool.fts3 import FTS3Transfertool
+from rucio.transfertool.globus import GlobusTransferTool
+from rucio.transfertool.mock import MockTransfertool
+from rucio.transfertool.transfertool import TransferStatusReport, Transfertool
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+    from typing import Any, Optional
+
+    from sqlalchemy.orm import Session
+
+    from rucio.common.types import InternalAccount
+    from rucio.core.topology import Topology
+    from rucio.rse.protocols.protocol import RSEProtocol
+
+    LoggerFunction = Callable[..., Any]
+
+"""
+The core transfer.py is specifically for handling transfer-requests, thus requests
+where the external_id is already known.
+Requests accessed by request_id  are covered in the core request.py
+"""
+
+REGION_ACCOUNTS = make_region().configure('dogpile.cache.memory', expiration_time=600)
+METRICS = MetricManager(module=__name__)
+
+WEBDAV_TRANSFER_MODE = config_get('conveyor', 'webdav_transfer_mode', False, None)
+
+DEFAULT_MULTIHOP_TOMBSTONE_DELAY = int(datetime.timedelta(hours=2).total_seconds())
+
+TRANSFERTOOL_CLASSES_BY_NAME: "dict[str, type[Transfertool]]" = {
+    FTS3Transfertool.external_name: FTS3Transfertool,
+    GlobusTransferTool.external_name: GlobusTransferTool,
+    MockTransfertool.external_name: MockTransfertool,
+    BittorrentTransfertool.external_name: BittorrentTransfertool,
+}
+
+
+class ProtocolFactory:
+    """
+    Creates and caches protocol objects. Allowing to reuse them.
+    """
+    def __init__(self):
+        self.protocols = {}
+
+    def protocol(self, rse: RseData, scheme: "Optional[str]", operation: str):
+        protocol_key = '%s_%s_%s' % (operation, rse.id, scheme)
+        protocol = self.protocols.get(protocol_key)
+        if not protocol:
+            protocol = rsemgr.create_protocol(rse.info, operation, scheme)
+            self.protocols[protocol_key] = protocol
+        return protocol
+
+
+class DirectTransferImplementation(DirectTransfer):
+    """
+    The configuration for a direct (non-multi-hop) transfer. It can be a multi-source transfer.
+
+    The class wraps the legacy dict-based transfer definition to maintain compatibility with existing code
+    during the migration.
+    """
+    def __init__(self, source: RequestSource, destination: TransferDestination, rws: RequestWithSources,
+                 protocol_factory: ProtocolFactory, operation_src: str, operation_dest: str):
+        super().__init__(sources=[source], rws=rws)
+        self.destination = destination
+
+        self.protocol_factory = protocol_factory
+        self.operation_src = operation_src
+        self.operation_dest = operation_dest
+
+        self._dest_url = None
+        self._source_urls = {}
+
+    def __str__(self):
+        return '{sources}--{request_id}->{destination}'.format(
+            sources=','.join([str(s.rse) for s in self.sources]),
+            request_id=self.rws.request_id or '',
+            destination=self.dst.rse
+        )
+
+    @property
+    def src(self) -> RequestSource:
+        return self.sources[0]
+
+    @property
+    def dst(self) -> TransferDestination:
+        return self.destination
+
+    @property
+    def dest_url(self) -> str:
+        if not self._dest_url:
+            self._dest_url = self._generate_dest_url(self.dst, self.rws, self.protocol_factory, self.operation_dest)
+        return self._dest_url
+
+    def source_url(self, source: RequestSource) -> str:
+        url = self._source_urls.get(source.rse)
+        if not url:
+            self._source_urls[source.rse] = url = self._generate_source_url(
+                source,
+                self.dst,
+                rws=self.rws,
+                protocol_factory=self.protocol_factory,
+                operation=self.operation_src
+            )
+        return url
+
+    def dest_protocol(self) -> "RSEProtocol":
+        return self.protocol_factory.protocol(self.dst.rse, self.dst.scheme, self.operation_dest)
+
+    def source_protocol(self, source: RequestSource) -> "RSEProtocol":
+        return self.protocol_factory.protocol(source.rse, source.scheme, self.operation_src)
+
+    @staticmethod
+    def __rewrite_source_url(source_url, source_sign_url, dest_sign_url, source_scheme):
+        """
+        Parametrize source url for some special cases of source and destination schemes
+        """
+        if dest_sign_url == 'gcs':
+            if source_scheme in ['davs', 'https']:
+                source_url += '?copy_mode=push'
+        elif dest_sign_url == 's3':
+            if source_scheme in ['davs', 'https']:
+                source_url += '?copy_mode=push'
+        elif WEBDAV_TRANSFER_MODE:
+            if source_scheme in ['davs', 'https']:
+                source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
+
+        source_sign_url_map = {'gcs': 'gclouds', 's3': 's3s'}
+        if source_sign_url in source_sign_url_map:
+            if source_url[:7] == 'davs://':
+                source_url = source_sign_url_map[source_sign_url] + source_url[4:]
+            if source_url[:8] == 'https://':
+                source_url = source_sign_url_map[source_sign_url] + source_url[5:]
+
+        if source_url[:12] == 'srm+https://':
+            source_url = 'srm' + source_url[9:]
+        return source_url
+
+    @staticmethod
+    def __rewrite_dest_url(dest_url, dest_sign_url):
+        """
+        Parametrize destination url for some special cases of destination schemes
+        """
+        if dest_sign_url == 'gcs':
+            dest_url = re.sub('davs', 'gclouds', dest_url)
+            dest_url = re.sub('https', 'gclouds', dest_url)
+        elif dest_sign_url == 's3':
+            dest_url = re.sub('davs', 's3s', dest_url)
+            dest_url = re.sub('https', 's3s', dest_url)
+
+        if dest_url[:12] == 'srm+https://':
+            dest_url = 'srm' + dest_url[9:]
+        return dest_url
+
+    @classmethod
+    def _generate_source_url(cls, src: RequestSource, dst: TransferDestination, rws: RequestWithSources, protocol_factory: ProtocolFactory, operation: str):
+        """
+        Generate the source url which will be used as origin to copy the file from request rws towards the given dst endpoint
+        """
+        # Get source protocol
+        protocol = protocol_factory.protocol(src.rse, src.scheme, operation)
+
+        # Compute the source URL
+        source_sign_url = src.rse.attributes.get(RseAttr.SIGN_URL, None)
+        dest_sign_url = dst.rse.attributes.get(RseAttr.SIGN_URL, None)
+        source_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name, 'path': src.file_path}).values())[0]
+        source_url = cls.__rewrite_source_url(source_url, source_sign_url=source_sign_url, dest_sign_url=dest_sign_url, source_scheme=src.scheme)
+        return source_url
+
+    @classmethod
+    def _generate_dest_url(cls, dst: TransferDestination, rws: RequestWithSources, protocol_factory: ProtocolFactory, operation: str):
+        """
+        Generate the destination url for copying the file of request rws
+        """
+        # Get destination protocol
+        protocol = protocol_factory.protocol(dst.rse, dst.scheme, operation)
+
+        if dst.rse.info['deterministic']:
+            dest_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name}).values())[0]
+        else:
+            # compute dest url in case of non deterministic
+            # naming convention, etc.
+            dsn = get_dsn(rws.scope, rws.name, rws.attributes.get('dsn', None))
+            # DQ2 path always starts with /, but prefix might not end with /
+            naming_convention = dst.rse.attributes.get(RseAttr.NAMING_CONVENTION, None)
+            if rws.scope.external is not None:
+                dest_path = construct_non_deterministic_pfn(dsn, rws.scope.external, rws.name, naming_convention)
+            if dst.rse.is_tape():
+                if rws.retry_count or rws.activity == 'Recovery':
+                    dest_path = '%s_%i' % (dest_path, int(time.time()))
+
+            dest_url = list(protocol.lfns2pfns(lfns={'scope': rws.scope.external, 'name': rws.name, 'path': dest_path}).values())[0]
+
+        dest_sign_url = dst.rse.attributes.get(RseAttr.SIGN_URL, None)
+        dest_url = cls.__rewrite_dest_url(dest_url, dest_sign_url=dest_sign_url)
+        return dest_url
+
+
+class StageinTransferImplementation(DirectTransferImplementation):
+    """
+    A definition of a transfer which triggers a stagein operation.
+        - The source and destination url are identical
+        - must be from TAPE to non-TAPE RSE
+        - can only have one source
+    """
+    def __init__(
+            self,
+            source: RequestSource,
+            destination: TransferDestination,
+            rws: RequestWithSources,
+            protocol_factory: ProtocolFactory,
+            operation_src: str,
+            operation_dest: str
+    ):
+        if not source.rse.is_tape() or destination.rse.is_tape():
+            # allow staging_required QoS RSE to be TAPE to TAPE for pin
+            if not destination.rse.attributes.get(RseAttr.STAGING_REQUIRED, None):
+                raise RucioException("Stageing request {} must be from TAPE to DISK rse. Got {} and {}.".format(rws, source, destination))
+        super().__init__(source, destination, rws, protocol_factory, operation_src, operation_dest)
+
+    @property
+    def dest_url(self) -> str:
+        if not self._dest_url:
+            self._dest_url = self.src.url if self.src.url else self._generate_source_url(self.src,
+                                                                                         self.dst,
+                                                                                         rws=self.rws,
+                                                                                         protocol_factory=self.protocol_factory,
+                                                                                         operation=self.operation_dest)
+        return self._dest_url
+
+    def source_url(self, source: RequestSource) -> str:
+        # Source and dest url is the same for stagein requests
+        return self.dest_url
+
+
+def transfer_path_str(transfer_path: "list[DirectTransfer]") -> str:
+    """
+    an implementation of __str__ for a transfer path, which is a list of direct transfers, so not really an object
+    """
+    if not transfer_path:
+        return 'empty transfer path'
+
+    multi_tt = False
+    if len({hop.rws.transfertool for hop in transfer_path if hop.rws.transfertool}) > 1:
+        # The path relies on more than one transfertool
+        multi_tt = True
+
+    if len(transfer_path) == 1:
+        return str(transfer_path[0])
+
+    path_str = str(transfer_path[0].src.rse)
+    for hop in transfer_path:
+        path_str += '--{request_id}{transfertool}->{destination}'.format(
+            request_id=hop.rws.request_id or '',
+            transfertool=':{}'.format(hop.rws.transfertool) if multi_tt else '',
+            destination=hop.dst.rse,
+        )
+    return path_str
+
+
+@transactional_session
+def mark_submitting(
+        transfer: "DirectTransfer",
+        external_host: str,
+        *,
+        logger: "Callable",
+        session: "Session",
+):
+    """
+    Mark a transfer as submitting
+
+    :param transfer:   A transfer object
+    :param session:    Database session to use.
+    """
+
+    log_str = 'PREPARING REQUEST %s DID %s:%s TO SUBMITTING STATE PREVIOUS %s FROM %s TO %s USING %s ' % (transfer.rws.request_id,
+                                                                                                          transfer.rws.scope,
+                                                                                                          transfer.rws.name,
+                                                                                                          transfer.rws.previous_attempt_id,
+                                                                                                          [transfer.source_url(s) for s in transfer.sources],
+                                                                                                          transfer.dest_url,
+                                                                                                          external_host)
+    logger(logging.DEBUG, "%s", log_str)
+
+    stmt = update(
+        models.Request
+    ).where(
+        models.Request.id == transfer.rws.request_id,
+        models.Request.state == RequestState.QUEUED
+    ).execution_options(
+        synchronize_session=False
+    ).values(
+        {
+            'state': RequestState.SUBMITTING,
+            'external_id': None,
+            'external_host': external_host,
+            'dest_url': transfer.dest_url,
+            'submitted_at': datetime.datetime.utcnow(),
+        }
+    )
+    rowcount = session.execute(stmt).rowcount
+
+    if rowcount == 0:
+        raise RequestNotFound("Failed to prepare transfer: request %s does not exist or is not in queued state" % transfer.rws)
+
+
+@transactional_session
+def ensure_db_sources(
+        transfer_path: "list[DirectTransfer]",
+        *,
+        logger: "Callable",
+        session: "Session",
+):
+    """
+    Ensure the needed DB source objects exist
+    """
+
+    desired_sources = []
+    for transfer in transfer_path:
+
+        for source in transfer.sources:
+            common_source_attrs = {
+                "scope": transfer.rws.scope,
+                "name": transfer.rws.name,
+                "rse_id": source.rse.id,
+                "dest_rse_id": transfer.dst.rse.id,
+                "ranking": source.ranking,
+                "bytes": transfer.rws.byte_count,
+                "url": transfer.source_url(source),
+                "is_using": True,
+            }
+
+            desired_sources.append({'request_id': transfer.rws.request_id, **common_source_attrs})
+            if len(transfer_path) > 1 and transfer is not transfer_path[-1]:
+                # For multihop transfers, each hop's source is also an initial transfer's source.
+                desired_sources.append({'request_id': transfer_path[-1].rws.request_id, **common_source_attrs})
+
+    for source in desired_sources:
+        stmt = update(
+            models.Source
+        ).where(
+            models.Source.request_id == source['request_id'],
+            models.Source.rse_id == source['rse_id']
+        ).execution_options(
+            synchronize_session=False
+        ).values(
+            is_using=True
+        )
+        src_rowcount = session.execute(stmt).rowcount
+        if src_rowcount == 0:
+            models.Source(**source).save(session=session, flush=False)
+
+
+@transactional_session
+def set_transfers_state(
+        transfers,
+        state: "RequestState",
+        submitted_at: datetime.datetime,
+        external_host: str,
+        external_id: str,
+        transfertool: str,
+        *,
+        session: "Session",
+        logger
+):
+    """
+    Update the transfer info of a request.
+    :param transfers:  Dictionary containing request transfer info.
+    :param session:    Database session to use.
+    """
+
+    logger(logging.INFO, 'Setting state(%s), transfertool(%s), external_host(%s) and eid(%s) for transfers: %s',
+           state.name, transfertool, external_host, external_id, ', '.join(t.rws.request_id for t in transfers))
+    try:
+        for transfer in transfers:
+            rws = transfer.rws
+            logger(logging.DEBUG, 'COPYING REQUEST %s DID %s:%s USING %s with state(%s) with eid(%s)' % (rws.request_id, rws.scope, rws.name, external_host, state, external_id))
+            stmt = update(
+                models.Request
+            ).where(
+                models.Request.id == transfer.rws.request_id,
+                models.Request.state == RequestState.SUBMITTING
+            ).execution_options(
+                synchronize_session=False
+            ).values(
+                {
+                    models.Request.state: state,
+                    models.Request.external_id: external_id,
+                    models.Request.external_host: external_host,
+                    models.Request.source_rse_id: transfer.src.rse.id,
+                    models.Request.submitted_at: submitted_at,
+                    models.Request.transfertool: transfertool,
+                }
+            )
+            rowcount = session.execute(stmt).rowcount
+
+            if rowcount == 0:
+                raise RucioException("%s: failed to set transfer state: request doesn't exist or is not in SUBMITTING state" % rws)
+
+            stmt = select(
+                models.DataIdentifier.datatype
+            ).where(
+                models.DataIdentifier.scope == rws.scope,
+                models.DataIdentifier.name == rws.name,
+            )
+            datatype = session.execute(stmt).scalar_one_or_none()
+
+            msg = {'request-id': rws.request_id,
+                   'request-type': rws.request_type,
+                   'scope': rws.scope.external,
+                   'name': rws.name,
+                   'dataset': None,
+                   'datasetScope': None,
+                   'src-rse-id': transfer.src.rse.id,
+                   'src-rse': transfer.src.rse.name,
+                   'dst-rse-id': transfer.dst.rse.id,
+                   'dst-rse': transfer.dst.rse.name,
+                   'state': state,
+                   'activity': rws.activity,
+                   'file-size': rws.byte_count,
+                   'bytes': rws.byte_count,
+                   'checksum-md5': rws.md5,
+                   'checksum-adler': rws.adler32,
+                   'external-id': external_id,
+                   'external-host': external_host,
+                   'queued_at': str(submitted_at),
+                   'datatype': datatype}
+            if rws.scope.vo != 'def':
+                msg['vo'] = rws.scope.vo
+
+            ds_scope = transfer.rws.attributes.get('ds_scope')
+            if ds_scope:
+                msg['datasetScope'] = ds_scope
+            ds_name = transfer.rws.attributes.get('ds_name')
+            if ds_name:
+                msg['dataset'] = ds_name
+
+            if msg['request-type']:
+                transfer_status = '%s-%s' % (msg['request-type'].name, msg['state'].name)
+            else:
+                transfer_status = 'transfer-%s' % msg['state']
+            transfer_status = transfer_status.lower()
+
+            message_core.add_message(transfer_status, msg, session=session)
+
+    except IntegrityError as error:
+        raise RucioException(error.args)
+
+    logger(logging.DEBUG, 'Finished to register transfer state for %s' % external_id)
+
+
+@transactional_session
+def update_transfer_state(
+        tt_status_report: TransferStatusReport,
+        stats_manager: request_core.TransferStatsManager,
+        *,
+        session: "Session",
+        logger=logging.log
+):
+    """
+    Used by poller and consumer to update the internal state of requests,
+    after the response by the external transfertool.
+
+    :param tt_status_report:      The transfertool status update, retrieved via request.query_request().
+    :param session:               The database session to use.
+    :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
+    :returns:                     The number of updated requests
+    """
+
+    request_id = tt_status_report.request_id
+    nb_updated = 0
+    try:
+        fields_to_update = tt_status_report.get_db_fields_to_update(session=session, logger=logger)
+        if not fields_to_update:
+            request_core.update_request(request_id, raise_on_missing=True, session=session)
+            return False
+        else:
+            logger(logging.INFO, 'UPDATING REQUEST %s FOR %s with changes: %s' % (str(request_id), tt_status_report, fields_to_update))
+
+            request = request_core.get_request(request_id, session=session)
+            updated = transition_request_state(request_id, request=request, session=session, **fields_to_update)
+
+            if not updated:
+                return nb_updated
+            nb_updated += 1
+
+            if tt_status_report.state == RequestState.FAILED:
+                if request_core.is_intermediate_hop(request):
+                    nb_updated += request_core.handle_failed_intermediate_hop(request, session=session)
+
+            if tt_status_report.state:
+                stats_manager.observe(
+                    src_rse_id=request['source_rse_id'],
+                    dst_rse_id=request['dest_rse_id'],
+                    activity=request['activity'],
+                    state=tt_status_report.state,
+                    file_size=request['bytes'],
+                    submitted_at=request.get('submitted_at', None),
+                    started_at=fields_to_update.get('started_at', None),
+                    transferred_at=fields_to_update.get('transferred_at', None),
+                    session=session,
+                )
+            request_core.add_monitor_message(
+                new_state=tt_status_report.state,
+                request=request,
+                additional_fields=tt_status_report.get_monitor_msg_fields(session=session, logger=logger),
+                session=session
+            )
+            return nb_updated
+    except UnsupportedOperation as error:
+        logger(logging.WARNING, "Request %s doesn't exist - Error: %s" % (request_id, str(error).replace('\n', '')))
+        return 0
+    except Exception:
+        logger(logging.CRITICAL, "Exception", exc_info=True)
+
+
+@transactional_session
+def mark_transfer_lost(request, *, session: "Session", logger=logging.log):
+    new_state = RequestState.LOST
+    reason = "The FTS job lost"
+
+    err_msg = request_core.get_transfer_error(new_state, reason)
+    transition_request_state(request['id'], state=new_state, external_id=request['external_id'], err_msg=err_msg, session=session, logger=logger)
+
+    request_core.add_monitor_message(new_state=new_state, request=request, additional_fields={'reason': reason}, session=session)
+
+
+@METRICS.count_it
+@transactional_session
+def touch_transfer(external_host, transfer_id, *, session: "Session"):
+    """
+    Update the timestamp of requests in a transfer. Fails silently if the transfer_id does not exist.
+    :param request_host:   Name of the external host.
+    :param transfer_id:    External transfer job id as a string.
+    :param session:        Database session to use.
+    """
+    try:
+        # don't touch it if it's already touched in 30 seconds
+        stmt = update(
+            models.Request
+        ).prefix_with(
+            "/*+ INDEX(REQUESTS REQUESTS_EXTERNALID_UQ) */", dialect='oracle'
+        ).where(
+            models.Request.external_id == transfer_id,
+            models.Request.state == RequestState.SUBMITTED,
+            models.Request.updated_at < datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
+        ).execution_options(
+            synchronize_session=False
+        ).values(
+            updated_at=datetime.datetime.utcnow()
+        )
+        session.execute(stmt)
+    except IntegrityError as error:
+        raise RucioException(error.args)
+
+
+def _create_transfer_definitions(
+        topology: "Topology",
+        protocol_factory: ProtocolFactory,
+        rws: RequestWithSources,
+        sources: "Iterable[RequestSource]",
+        max_sources: int,
+        multi_source_sources: "Iterable[RequestSource]",
+        limit_dest_schemes: list[str],
+        operation_src: str,
+        operation_dest: str,
+        domain: str,
+        *,
+        session: "Session",
+) -> "dict[RseData, list[DirectTransfer]]":
+    """
+    Find the all paths from sources towards the destination of the given transfer request.
+    Create the transfer definitions for each point-to-point transfer (multi-source, when possible)
+    """
+    shortest_paths = topology.search_shortest_paths(src_nodes=[s.rse for s in sources], dst_node=rws.dest_rse,
+                                                    operation_src=operation_src, operation_dest=operation_dest,
+                                                    domain=domain, limit_dest_schemes=limit_dest_schemes, session=session)
+
+    transfers_by_source = {}
+    sources_by_rse = {s.rse: s for s in sources}
+    paths_by_source = {sources_by_rse[rse]: path for rse, path in shortest_paths.items()}
+    for source, list_hops in paths_by_source.items():
+        transfer_path = []
+        for hop in list_hops:
+            hop_src_rse = hop['source_rse']
+            hop_dst_rse = hop['dest_rse']
+            src = RequestSource(
+                rse=hop_src_rse,
+                file_path=source.file_path if hop_src_rse == source.rse else None,
+                ranking=source.ranking if hop_src_rse == source.rse else 0,
+                distance=hop['cumulated_distance'] if hop_src_rse == source.rse else hop['hop_distance'],
+                scheme=hop['source_scheme'],
+            )
+            dst = TransferDestination(
+                rse=hop_dst_rse,
+                scheme=hop['dest_scheme'],
+            )
+            hop_definition = DirectTransferImplementation(
+                source=src,
+                destination=dst,
+                operation_src=operation_src,
+                operation_dest=operation_dest,
+                # keep the current rws for last hop; create a new one for other hops
+                rws=rws if hop_dst_rse == rws.dest_rse else RequestWithSources(
+                    id_=None,
+                    request_type=rws.request_type,
+                    rule_id=None,
+                    scope=rws.scope,
+                    name=rws.name,
+                    md5=rws.md5,
+                    adler32=rws.adler32,
+                    byte_count=rws.byte_count,
+                    activity=rws.activity,
+                    attributes={
+                        'activity': rws.activity,
+                        'source_replica_expression': None,
+                        'lifetime': None,
+                        'ds_scope': None,
+                        'ds_name': None,
+                        'bytes': rws.byte_count,
+                        'md5': rws.md5,
+                        'adler32': rws.adler32,
+                        'priority': None,
+                        'allow_tape_source': True
+                    },
+                    previous_attempt_id=None,
+                    dest_rse=hop_dst_rse,
+                    account=rws.account,
+                    retry_count=0,
+                    priority=rws.priority,
+                    transfertool=rws.transfertool,
+                ),
+                protocol_factory=protocol_factory,
+            )
+
+            transfer_path.append(hop_definition)
+        transfers_by_source[source.rse] = transfer_path
+
+    # create multi-source transfers: add additional sources if possible
+    for transfer_path in transfers_by_source.values():
+        if len(transfer_path) == 1 and not transfer_path[0].src.rse.is_tape():
+            # Multiple single-hop DISK rses can be used together in "multi-source" transfers
+            #
+            # Try adding additional single-hop DISK rses sources to the transfer
+            main_source_schemes = __add_compatible_schemes(schemes=[transfer_path[0].dst.scheme], allowed_schemes=SUPPORTED_PROTOCOLS)
+            added_sources = 0
+            for source in sorted(multi_source_sources, key=lambda s: (-s.ranking, s.distance)):
+                if added_sources >= max_sources:
+                    break
+
+                edge = topology.edge(source.rse, transfer_path[0].dst.rse)
+                if not edge:
+                    # There is no direct connection between this source and the destination
+                    continue
+
+                if source.rse == transfer_path[0].src.rse:
+                    # This is the main source. Don't add a duplicate.
+                    continue
+
+                if source.rse.is_tape():
+                    continue
+
+                try:
+                    matching_scheme = rsemgr.find_matching_scheme(
+                        rse_settings_src=source.rse.info,
+                        rse_settings_dest=transfer_path[0].dst.rse.info,
+                        operation_src=operation_src,
+                        operation_dest=operation_dest,
+                        domain=domain,
+                        scheme=main_source_schemes)
+                except RSEProtocolNotSupported:
+                    continue
+
+                transfer_path[0].sources.append(
+                    RequestSource(
+                        rse=source.rse,
+                        file_path=source.file_path,
+                        ranking=source.ranking,
+                        distance=edge.cost,
+                        scheme=matching_scheme[1],
+                    )
+                )
+                added_sources += 1
+    return transfers_by_source
+
+
+def _create_stagein_definitions(
+        rws: RequestWithSources,
+        sources: "Iterable[RequestSource]",
+        limit_dest_schemes: list[str],
+        operation_src: str,
+        operation_dest: str,
+        protocol_factory: ProtocolFactory,
+) -> "dict[RseData, list[DirectTransfer]]":
+    """
+    for each source, create a single-hop transfer path with a one stageing definition inside
+    """
+    transfers_by_source = {
+        source.rse: [
+            cast(DirectTransfer, StageinTransferImplementation(
+                source=RequestSource(
+                    rse=source.rse,
+                    file_path=source.file_path,
+                    url=source.url,
+                    scheme=limit_dest_schemes,  # type: ignore (list passed instead of single scheme)
+                ),
+                destination=TransferDestination(
+                    rse=rws.dest_rse,
+                    scheme=limit_dest_schemes,  # type: ignore (list passed instead of single scheme)
+                ),
+                operation_src=operation_src,
+                operation_dest=operation_dest,
+                rws=rws,
+                protocol_factory=protocol_factory,
+            ))
+
+        ]
+        for source in sources
+    }
+    return transfers_by_source
+
+
+def get_dsn(scope, name, dsn):
+    if dsn:
+        return dsn
+    # select a containing dataset
+    for parent in did.list_parent_dids(scope, name):
+        if parent['type'] == DIDType.DATASET:
+            return parent['name']
+    return 'other'
+
+
+def __compress_multihops(
+        paths_by_source: "Iterable[tuple[RequestSource, Sequence[DirectTransfer]]]",
+        sources: "Iterable[RequestSource]",
+) -> "Iterator[tuple[RequestSource, Sequence[DirectTransfer]]]":
+    # Compress multihop transfers which contain other sources as part of itself.
+    # For example: multihop A->B->C and B is a source, compress A->B->C into B->C
+    source_rses = {s.rse.id for s in sources}
+    seen_source_rses = set()
+    for source, path in paths_by_source:
+        if len(path) > 1:
+            # find the index of the first hop starting from the end which is also a source. Path[0] will always be a source.
+            last_source_idx = next((idx for idx, hop in reversed(list(enumerate(path))) if hop.src.rse.id in source_rses), (0, None))
+            if last_source_idx > 0:
+                path = path[last_source_idx:]
+
+        # Deduplicate paths from same source
+        src_rse_id = path[0].src.rse.id
+        if src_rse_id not in seen_source_rses:
+            seen_source_rses.add(src_rse_id)
+            yield source, path
+
+
+class TransferPathBuilder:
+    def __init__(
+            self,
+            topology: "Topology",
+            protocol_factory: ProtocolFactory,
+            max_sources: int,
+            preparer_mode: bool = False,
+            schemes: "Optional[list[str]]" = None,
+            failover_schemes: "Optional[list[str]]" = None,
+            requested_source_only: bool = False,
+    ):
+        self.failover_schemes = failover_schemes if failover_schemes is not None else []
+        self.schemes = schemes if schemes is not None else []
+        self.topology = topology
+        self.preparer_mode = preparer_mode
+        self.protocol_factory = protocol_factory
+        self.max_sources = max_sources
+        self.requested_source_only = requested_source_only
+
+        self.definition_by_request_id = {}
+
+    def build_or_return_cached(
+            self,
+            rws: RequestWithSources,
+            sources: "Iterable[RequestSource]",
+            *,
+            logger: "LoggerFunction" = logging.log,
+            session: "Session"
+    ) -> "Mapping[RseData, Sequence[DirectTransfer]]":
+        """
+        Warning: The function currently caches the result for the given request and returns it for later calls
+        with the same request id. As a result: it can return more (or less) sources than what is provided in the
+        `sources` argument. This is done for performance reasons. As of time of writing, this behavior is not problematic
+        for the callers of this method.
+        """
+        definition = self.definition_by_request_id.get(rws.request_id)
+        if definition:
+            return definition
+
+        transfer_schemes = self.schemes
+        if rws.previous_attempt_id and self.failover_schemes:
+            transfer_schemes = self.failover_schemes
+
+        candidate_sources = sources
+        if self.requested_source_only and rws.requested_source:
+            candidate_sources = [rws.requested_source] if rws.requested_source in sources else []
+
+        if rws.request_type == RequestType.STAGEIN:
+            definition = _create_stagein_definitions(
+                rws=rws,
+                sources=sources,
+                limit_dest_schemes=transfer_schemes,
+                operation_src='read',
+                operation_dest='write',
+                protocol_factory=self.protocol_factory
+            )
+        else:
+            definition = _create_transfer_definitions(
+                topology=self.topology,
+                rws=rws,
+                sources=candidate_sources,
+                max_sources=self.max_sources,
+                multi_source_sources=[] if self.preparer_mode else sources,
+                limit_dest_schemes=transfer_schemes,
+                operation_src='third_party_copy_read',
+                operation_dest='third_party_copy_write',
+                domain='wan',
+                protocol_factory=self.protocol_factory,
+                session=session
+            )
+        self.definition_by_request_id[rws.request_id] = definition
+        return definition
+
+
+class _SkipSource:
+    pass
+
+
+SKIP_SOURCE = _SkipSource()
+
+
+class RequestRankingContext:
+    """
+    Helper class used by SourceRankingStrategy. It allows to store additional request-specific
+    context data and access it when handling a specific source of the given request.
+    """
+
+    def __init__(self, strategy: "SourceRankingStrategy", rws: "RequestWithSources"):
+        self.strategy = strategy
+        self.rws = rws
+
+    def apply(self, source: RequestSource) -> "int | _SkipSource":
+        verdict = self.strategy.apply(self, source)
+        if verdict is None:
+            verdict = sys.maxsize
+        return verdict
+
+
+class SourceRankingStrategy:
+    """
+    Represents a source ranking strategy. Used to order the sources of a request and decide
+    which will be the actual source used for the transfer.
+
+    If filter_only is True, any value other than SKIP_SOURCE returned by apply() will be ignored.
+    """
+    filter_only: bool = False
+
+    def for_request(
+            self,
+            rws: RequestWithSources,
+            sources: "Iterable[RequestSource]",
+            *,
+            logger: "LoggerFunction" = logging.log,
+            session: "Session"
+    ) -> "RequestRankingContext":
+        return RequestRankingContext(self, rws)
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        """
+        Normally, this function will be called indirectly, via self.for_request(...).apply(source).
+
+        It is expected to either return SKIP_SOURCE to signal that this source must be ignored;
+        or an integer which gives the cost of the given source under the current strategy
+        (smaller cost: higher priority).
+        If `None` is returned, it will be interpreted as sys.maxsize (i.e. very low priority).
+        This is done to avoid requiring an explicit integer in filter-only strategies.
+        """
+        pass
+
+    class _ClassNameDescriptor:
+        """
+        Automatically set the external_name of the strategy to the class name.
+        """
+        def __get__(self, obj, objtype=None):
+            if objtype is not None:
+                return objtype.__name__
+            return type(obj).__name__
+
+    external_name = _ClassNameDescriptor()
+
+
+class SourceFilterStrategy(SourceRankingStrategy):
+    filter_only = True
+
+
+class EnforceSourceRSEExpression(SourceFilterStrategy):
+
+    class _RankingContext(RequestRankingContext):
+        def __init__(self, strategy: "SourceRankingStrategy", rws: "RequestWithSources", allowed_source_rses: "Optional[set[str]]"):
+            super().__init__(strategy, rws)
+            self.allowed_source_rses = allowed_source_rses
+
+    def for_request(
+            self,
+            rws: RequestWithSources,
+            sources: "Iterable[RequestSource]",
+            *,
+            logger: "LoggerFunction" = logging.log,
+            session: "Session"
+    ) -> "RequestRankingContext":
+        # parse source expression
+        allowed_source_rses = None
+        source_replica_expression = rws.attributes.get('source_replica_expression', None)
+        if source_replica_expression:
+            try:
+                parsed_rses = parse_expression(source_replica_expression, session=session)
+            except InvalidRSEExpression as error:
+                logger(logging.ERROR, "%s: Invalid RSE exception %s: %s", rws.request_id, source_replica_expression, str(error))
+                allowed_source_rses = set()
+            else:
+                allowed_source_rses = {x['id'] for x in parsed_rses}
+        return self._RankingContext(self, rws, allowed_source_rses)
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        ctx = cast(EnforceSourceRSEExpression._RankingContext, ctx)
+        if ctx.allowed_source_rses is not None and source.rse.id not in ctx.allowed_source_rses:
+            return SKIP_SOURCE
+
+
+class SkipRestrictedRSEs(SourceFilterStrategy):
+
+    def __init__(self, admin_accounts: "Optional[set[InternalAccount]]" = None):
+        super().__init__()
+        self.admin_accounts = admin_accounts if admin_accounts is not None else []
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        if source.rse.attributes.get(RseAttr.RESTRICTED_READ) and ctx.rws.account not in self.admin_accounts:
+            return SKIP_SOURCE
+
+
+class SkipBlocklistedRSEs(SourceFilterStrategy):
+
+    def __init__(self, topology: "Topology"):
+        super().__init__()
+        self.topology = topology
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        # Ignore blocklisted RSEs
+        if not source.rse.columns['availability_read'] and not self.topology.ignore_availability:
+            return SKIP_SOURCE
+
+
+class EnforceStagingBuffer(SourceFilterStrategy):
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        # For staging requests, the staging_buffer attribute must be correctly set
+        if ctx.rws.request_type == RequestType.STAGEIN and source.rse.attributes.get(RseAttr.STAGING_BUFFER) != ctx.rws.dest_rse.name:
+            return SKIP_SOURCE
+
+
+class RestrictTapeSources(SourceFilterStrategy):
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        # Ignore tape sources if they are not desired
+        if source.rse.is_tape_or_staging_required() and not ctx.rws.attributes.get("allow_tape_source", True):
+            return SKIP_SOURCE
+
+
+class HighestAdjustedRankingFirst(SourceRankingStrategy):
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        source_ranking_penalty = 1 if source.rse.is_tape_or_staging_required() else 0
+        return - source.ranking + source_ranking_penalty
+
+
+class PreferDiskOverTape(SourceRankingStrategy):
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        return int(source.rse.is_tape_or_staging_required())  # rely on the fact that False < True
+
+
+class PathDistance(SourceRankingStrategy):
+
+    class _RankingContext(RequestRankingContext):
+        def __init__(self, strategy: "SourceRankingStrategy", rws: "RequestWithSources", paths_for_rws: "Mapping[RseData, Sequence[DirectTransfer]]"):
+            super().__init__(strategy, rws)
+            self.paths_for_rws = paths_for_rws
+
+    def __init__(self, transfer_path_builder: TransferPathBuilder):
+        super().__init__()
+        self.transfer_path_builder = transfer_path_builder
+
+    def for_request(
+            self,
+            rws: RequestWithSources,
+            sources: "Iterable[RequestSource]",
+            *,
+            logger: "LoggerFunction" = logging.log,
+            session: "Session"
+    ) -> "RequestRankingContext":
+        paths_for_rws = self.transfer_path_builder.build_or_return_cached(rws, sources, logger=logger, session=session)
+        return PathDistance._RankingContext(self, rws, paths_for_rws)
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        if not path:
+            return SKIP_SOURCE
+        return path[0].src.distance
+
+
+class PreferSingleHop(PathDistance):
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        if not path:
+            return SKIP_SOURCE
+        return int(len(path) > 1)
+
+
+class FailureRate(SourceRankingStrategy):
+    """
+    A source ranking strategy that ranks source nodes based on their failure rates for the past hour. Failure rate is
+    calculated by dividing files failed by files attempted.
+    """
+    class _FailureRateStat:
+        def __init__(self) -> None:
+            self.files_done = 0
+            self.files_failed = 0
+
+        def incorporate_stat(self, stat: "Mapping[str, int]") -> None:
+            self.files_done += stat['files_done']
+            self.files_failed += stat['files_failed']
+
+        def get_failure_rate(self) -> int:
+            files_attempted = self.files_done + self.files_failed
+
+            # If no files have been sent yet, return failure rate as 0
+            if files_attempted == 0:
+                return 0
+
+            return int((self.files_failed / files_attempted) * 10000)
+
+    def __init__(self, stats_manager: "request_core.TransferStatsManager") -> None:
+        super().__init__()
+        self.source_stats = {}
+
+        for stat in stats_manager.load_totals(
+            datetime.datetime.utcnow() - datetime.timedelta(hours=1),
+            by_activity=False
+        ):
+            self.source_stats.setdefault(stat['src_rse_id'], self._FailureRateStat()).incorporate_stat(stat)
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        failure_rate = cast(FailureRate, ctx.strategy).source_stats.get(source.rse.id, self._FailureRateStat()).get_failure_rate()
+        return failure_rate
+
+
+class SkipSchemeMissmatch(PathDistance):
+    filter_only = True
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        # path == None means that there is no path;
+        # path == [] means that a path exists (according to distances) but cannot be used (scheme mismatch)
+        if path is not None and not path:
+            return SKIP_SOURCE
+
+
+class SkipIntermediateTape(PathDistance):
+    filter_only = True
+
+    def apply(self, ctx: RequestRankingContext, source: RequestSource) -> "Optional[int | _SkipSource]":
+        # Discard multihop transfers which contain a tape source as an intermediate hop
+        path = cast(PathDistance._RankingContext, ctx).paths_for_rws.get(source.rse)
+        if path and any(transfer.src.rse.is_tape_or_staging_required() for transfer in path[1:]):
+            return SKIP_SOURCE
+
+
+@transactional_session
+def build_transfer_paths(
+        topology: "Topology",
+        protocol_factory: "ProtocolFactory",
+        requests_with_sources: "Iterable[RequestWithSources]",
+        admin_accounts: "Optional[set[InternalAccount]]" = None,
+        schemes: "Optional[list[str]]" = None,
+        failover_schemes: "Optional[list[str]]" = None,
+        max_sources: int = 4,
+        transfertools: "Optional[list[str]]" = None,
+        requested_source_only: bool = False,
+        preparer_mode: bool = False,
+        *,
+        session: "Session",
+        logger: "Callable" = logging.log,
+):
+    """
+    For each request, find all possible transfer paths from its sources, which respect the
+    constraints enforced by the request (attributes, type, etc) and the arguments of this function
+
+    build a multi-source transfer if possible: The scheme compatibility is important for multi-source transfers.
+    We iterate again over the single-hop sources and build a new transfer definition while enforcing the scheme compatibility
+    with the initial source.
+
+    Each path is a list of hops. Each hop is a transfer definition.
+    """
+    transfer_path_builder = TransferPathBuilder(
+        topology=topology,
+        schemes=schemes,
+        failover_schemes=failover_schemes,
+        protocol_factory=protocol_factory,
+        max_sources=max_sources,
+        preparer_mode=preparer_mode,
+        requested_source_only=requested_source_only,
+    )
+
+    stats_manager = request_core.TransferStatsManager()
+
+    available_strategies = {
+        EnforceSourceRSEExpression.external_name: lambda: EnforceSourceRSEExpression(),
+        SkipBlocklistedRSEs.external_name: lambda: SkipBlocklistedRSEs(topology=topology),
+        SkipRestrictedRSEs.external_name: lambda: SkipRestrictedRSEs(admin_accounts=admin_accounts),
+        EnforceStagingBuffer.external_name: lambda: EnforceStagingBuffer(),
+        RestrictTapeSources.external_name: lambda: RestrictTapeSources(),
+        SkipSchemeMissmatch.external_name: lambda: SkipSchemeMissmatch(transfer_path_builder=transfer_path_builder),
+        SkipIntermediateTape.external_name: lambda: SkipIntermediateTape(transfer_path_builder=transfer_path_builder),
+        HighestAdjustedRankingFirst.external_name: lambda: HighestAdjustedRankingFirst(),
+        PreferDiskOverTape.external_name: lambda: PreferDiskOverTape(),
+        PathDistance.external_name: lambda: PathDistance(transfer_path_builder=transfer_path_builder),
+        PreferSingleHop.external_name: lambda: PreferSingleHop(transfer_path_builder=transfer_path_builder),
+        FailureRate.external_name: lambda: FailureRate(stats_manager=stats_manager),
+    }
+
+    default_strategies = [
+        EnforceSourceRSEExpression.external_name,
+        SkipBlocklistedRSEs.external_name,
+        SkipRestrictedRSEs.external_name,
+        EnforceStagingBuffer.external_name,
+        RestrictTapeSources.external_name,
+        # Without the SkipSchemeMissmatch strategy, requests will never be transitioned to the
+        # RequestState.MISMATCH_SCHEME state. It _MUST_ be placed before the other Path-based strategies.
+        SkipSchemeMissmatch.external_name,
+        SkipIntermediateTape.external_name,
+        HighestAdjustedRankingFirst.external_name,
+        PreferDiskOverTape.external_name,
+        PathDistance.external_name,
+        PreferSingleHop.external_name,
+    ]
+    strategy_names = config_get_list('transfers', 'source_ranking_strategies', default=default_strategies)
+
+    try:
+        strategies = list(available_strategies[name]() for name in strategy_names)
+    except KeyError:
+        logger(logging.ERROR, "One of the configured source_ranking_strategies doesn't exist %s", strategy_names, exc_info=True)
+        raise
+
+    if admin_accounts is None:
+        admin_accounts = set()
+
+    # Do not print full source RSE list for DIDs which have many sources. Otherwise we fill the monitoring
+    # storage with data which has little to no benefit. This log message is unlikely to help debugging
+    # transfers issues when there are many sources, but can be very useful for small number of sources.
+    num_sources_in_logs = 4
+
+    candidate_paths_by_request_id, reqs_no_source, reqs_only_tape_source, reqs_scheme_mismatch = {}, set(), set(), set()
+    reqs_unsupported_transfertool = set()
+    for rws in requests_with_sources:
+
+        rws.dest_rse.ensure_loaded(load_name=True, load_info=True, load_attributes=True, load_columns=True, session=session)
+        all_sources = rws.sources
+        for source in all_sources:
+            source.rse.ensure_loaded(load_name=True, load_info=True, load_attributes=True, load_columns=True, session=session)
+
+        # Assume request doesn't have any sources. Will be removed later if sources are found.
+        reqs_no_source.add(rws.request_id)
+        if not all_sources:
+            logger(logging.INFO, '%s: has no sources. Skipping.', rws)
+            continue
+
+        logger(logging.DEBUG, '%s: Working on %d sources%s: %s%s',
+               rws,
+               len(all_sources),
+               f' (priority {rws.requested_source.rse})' if requested_source_only and rws.requested_source else '',
+               ','.join('{}:{}:{}'.format(src.rse, src.ranking, src.distance) for src in all_sources[:num_sources_in_logs]),
+               '... and %d others' % (len(all_sources) - num_sources_in_logs) if len(all_sources) > num_sources_in_logs else '')
+
+        # Check if destination is blocked
+        if not (topology.ignore_availability or rws.dest_rse.columns['availability_write']):
+            logger(logging.WARNING, '%s: dst RSE is blocked for write. Will skip the submission of new jobs', rws.request_id)
+            continue
+        if rws.account not in admin_accounts and rws.dest_rse.attributes.get(RseAttr.RESTRICTED_WRITE):
+            logger(logging.WARNING, '%s: dst RSE is restricted for write. Will skip the submission', rws.request_id)
+            continue
+
+        if rws.transfertool and transfertools and rws.transfertool not in transfertools:
+            # The request explicitly asks for a transfertool which this submitter doesn't support
+            logger(logging.INFO, '%s: unsupported transfertool. Skipping.', rws.request_id)
+            reqs_unsupported_transfertool.add(rws.request_id)
+            reqs_no_source.remove(rws.request_id)
+            continue
+
+        # For each strategy name, gives the sources which were rejected by it
+        rejected_sources = defaultdict(list)
+        # Cost of each accepted source (lists of ordered costs: one for each ranking strategy)
+        cost_vectors = {s: [] for s in rws.sources}
+        for strategy in strategies:
+            sources = list(cost_vectors)
+            if not sources:
+                # All sources where filtered by previous strategies. It's worthless to continue.
+                break
+            rws_strategy = strategy.for_request(rws, sources, logger=logger, session=session)
+            for source in sources:
+                verdict = rws_strategy.apply(source)
+                if verdict is SKIP_SOURCE:
+                    rejected_sources[strategy.external_name].append(source)
+                    cost_vectors.pop(source)
+                elif not strategy.filter_only:
+                    cost_vectors[source].append(verdict)
+
+        transfers_by_rse = transfer_path_builder.build_or_return_cached(rws, cost_vectors, logger=logger, session=session)
+        candidate_paths = ((s, transfers_by_rse[s.rse]) for s, _ in sorted(cost_vectors.items(), key=operator.itemgetter(1)))
+        if not preparer_mode:
+            candidate_paths = __compress_multihops(candidate_paths, all_sources)
+        candidate_paths = list(candidate_paths)
+
+        ordered_sources_log = ', '.join(
+            f"{s.rse}:{':'.join(str(e) for e in cost_vectors[s])}"
+            f"{'(actual source ' + str(path[0].src.rse) + ')' if s.rse != path[0].src.rse else ''}"
+            f"{'(multihop)' if len(path) > 1 else ''}"
+            for s, path in candidate_paths[:num_sources_in_logs]
+        )
+        if len(candidate_paths) > num_sources_in_logs:
+            ordered_sources_log += '... and %d others' % (len(candidate_paths) - num_sources_in_logs)
+        filtered_rses_log = ''
+        for strategy_name, sources in rejected_sources.items():
+            filtered_rses_log += f'; {len(sources)} dropped by strategy "{strategy_name}": '
+            filtered_rses_log += ','.join(str(s.rse) for s in sources[:num_sources_in_logs])
+            if len(sources) > num_sources_in_logs:
+                filtered_rses_log += '... and %d others' % (len(sources) - num_sources_in_logs)
+        logger(logging.INFO, '%s: %d ordered sources: %s%s', rws, len(candidate_paths), ordered_sources_log, filtered_rses_log)
+
+        if not candidate_paths:
+            # It can happen that some sources are skipped because they are TAPE, and others because
+            # of scheme mismatch. However, we can only have one state in the database. I picked to
+            # prioritize setting only_tape_source without any particular reason.
+            if RestrictTapeSources.external_name in rejected_sources:
+                logger(logging.DEBUG, '%s: Only tape sources found' % rws.request_id)
+                reqs_only_tape_source.add(rws.request_id)
+                reqs_no_source.remove(rws.request_id)
+            elif SkipSchemeMissmatch.external_name in rejected_sources:
+                logger(logging.DEBUG, '%s: Scheme mismatch detected' % rws.request_id)
+                reqs_scheme_mismatch.add(rws.request_id)
+                reqs_no_source.remove(rws.request_id)
+            else:
+                logger(logging.DEBUG, '%s: No candidate path found' % rws.request_id)
+            continue
+
+        candidate_paths_by_request_id[rws.request_id] = [path for _, path in candidate_paths]
+        reqs_no_source.remove(rws.request_id)
+
+    return candidate_paths_by_request_id, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source, reqs_unsupported_transfertool
+
+
+def __add_compatible_schemes(schemes, allowed_schemes):
+    """
+    Add the compatible schemes to a list of schemes
+    :param schemes:           Schemes as input.
+    :param allowed_schemes:   Allowed schemes, only these can be in the output.
+    :returns:                 List of schemes
+    """
+
+    return_schemes = []
+    for scheme in schemes:
+        if scheme in allowed_schemes:
+            return_schemes.append(scheme)
+            for scheme_map_scheme in constants.SCHEME_MAP.get(scheme, []):
+                if scheme_map_scheme not in allowed_schemes:
+                    continue
+                else:
+                    return_schemes.append(scheme_map_scheme)
+    return list(set(return_schemes))
+
+
+@read_session
+def list_transfer_admin_accounts(*, session: "Session") -> "set[InternalAccount]":
+    """
+    List admin accounts and cache the result in memory
+    """
+
+    result = REGION_ACCOUNTS.get('transfer_admin_accounts')
+    if isinstance(result, NoValue):
+        result = [acc['account'] for acc in list_accounts(filter_={'admin': True}, session=session)]
+        REGION_ACCOUNTS.set('transfer_admin_accounts', result)
+    return set(result)
+
+
+def update_transfer_priority(transfers_to_update, logger=logging.log):
+    """
+    Update transfer priority in fts
+
+    :param transfers_to_update: dict {external_host1: {transfer_id1: priority, transfer_id2: priority, ...}, ...}
+    :param logger: decorated logger instance
+    """
+
+    for external_host, priority_by_transfer_id in transfers_to_update.items():
+        transfertool_obj = FTS3Transfertool(external_host=external_host)
+        for transfer_id, priority in priority_by_transfer_id.items():
+            res = transfertool_obj.update_priority(transfer_id=transfer_id, priority=priority)
+            logger(logging.DEBUG, "Updated transfer %s priority in transfertool to %s: %s" % (transfer_id, priority, res['http_message']))
+
+
+def cancel_transfers(transfers_to_cancel, logger=logging.log):
+    """
+    Cancel transfers in fts
+
+    :param transfers_to_cancel: dict {external_host1: {transfer_id1, transfer_id2}, external_host2: [...], ...}
+    :param logger: decorated logger instance
+    """
+
+    for external_host, transfer_ids in transfers_to_cancel.items():
+        transfertool_obj = FTS3Transfertool(external_host=external_host)
+        for transfer_id in transfer_ids:
+            try:
+                transfertool_obj.cancel(transfer_ids=[transfer_id])
+                logger(logging.DEBUG, "Cancelled FTS3 transfer %s on %s" % (transfer_id, transfertool_obj))
+            except Exception as error:
+                logger(logging.WARNING, 'Could not cancel FTS3 transfer %s on %s: %s' % (transfer_id, transfertool_obj, str(error)))
+
+
+@METRICS.count_it
+def cancel_transfer(transfertool_obj, transfer_id):
+    """
+    Cancel a transfer based on external transfer id.
+
+    :param transfertool_obj: Transfertool object to be used for cancellation.
+    :param transfer_id:      External-ID as a 32 character hex string.
+    """
+
+    try:
+        transfertool_obj.cancel(transfer_ids=[transfer_id])
+    except Exception:
+        raise RucioException('Could not cancel FTS3 transfer %s on %s: %s' % (transfer_id, transfertool_obj, traceback.format_exc()))
+
+
+@transactional_session
+def prepare_transfers(
+        candidate_paths_by_request_id: "dict[str, list[list[DirectTransfer]]]",
+        logger: "LoggerFunction" = logging.log,
+        transfertools: "Optional[list[str]]" = None,
+        *,
+        session: "Session",
+) -> tuple[list[str], list[str]]:
+    """
+    Update transfer requests according to preparer settings.
+    """
+
+    reqs_no_transfertool = []
+    updated_reqs = []
+    for request_id, candidate_paths in candidate_paths_by_request_id.items():
+        selected_source = None
+        transfertool = None
+        rws = candidate_paths[0][-1].rws
+
+        for candidate_path in candidate_paths:
+            source = candidate_path[0].src
+            all_hops_ok = True
+            transfertool = None
+            for hop in candidate_path:
+                common_transfertools = get_supported_transfertools(hop.src.rse, hop.dst.rse, transfertools=transfertools, session=session)
+                if not common_transfertools:
+                    all_hops_ok = False
+                    break
+                # We need the last hop transfertool. Always prioritize fts3 if it exists.
+                transfertool = 'fts3' if 'fts3' in common_transfertools else common_transfertools.pop()
+
+            if all_hops_ok and transfertool:
+                selected_source = source
+                break
+
+        if not selected_source:
+            reqs_no_transfertool.append(request_id)
+            logger(logging.WARNING, '%s: all available sources were filtered', rws)
+            continue
+
+        update_dict: "dict[Any, Any]" = {
+            models.Request.state.name: _throttler_request_state(
+                activity=rws.activity,
+                source_rse=selected_source.rse,
+                dest_rse=rws.dest_rse,
+                session=session,
+            ),
+            models.Request.source_rse_id.name: selected_source.rse.id,
+        }
+        if transfertool:
+            update_dict[models.Request.transfertool.name] = transfertool
+
+        request_core.update_request(rws.request_id, session=session, **update_dict)
+        updated_reqs.append(request_id)
+
+    return updated_reqs, reqs_no_transfertool
+
+
+@stream_session
+def applicable_rse_transfer_limits(
+        source_rse: "Optional[RseData]" = None,
+        dest_rse: "Optional[RseData]" = None,
+        activity: "Optional[str]" = None,
+        *,
+        session: "Session",
+):
+    """
+    Find all RseTransferLimits which must be enforced for transfers between source and destination RSEs for the given activity.
+    """
+    source_limits = {}
+    if source_rse:
+        source_limits = source_rse.ensure_loaded(load_transfer_limits=True, session=session).transfer_limits.get(TransferLimitDirection.SOURCE, {})
+    dest_limits = {}
+    if dest_rse:
+        dest_limits = dest_rse.ensure_loaded(load_transfer_limits=True, session=session).transfer_limits.get(TransferLimitDirection.DESTINATION, {})
+
+    if activity is not None:
+        limit = source_limits.get(activity)
+        if limit:
+            yield limit
+
+        limit = dest_limits.get(activity)
+        if limit:
+            yield limit
+
+    # get "all_activities" limits
+    limit = source_limits.get(None)
+    if limit:
+        yield limit
+
+    limit = dest_limits.get(None)
+    if limit:
+        yield limit
+
+
+def _throttler_request_state(activity, source_rse, dest_rse, *, session: "Session") -> RequestState:
+    """
+    Takes request attributes to return a new state for the request
+    based on throttler settings. Always returns QUEUED,
+    if the throttler mode is not set.
+    """
+    limit_found = False
+    if any(applicable_rse_transfer_limits(activity=activity, source_rse=source_rse, dest_rse=dest_rse, session=session)):
+        limit_found = True
+
+    return RequestState.WAITING if limit_found else RequestState.QUEUED
+
+
+@read_session
+def get_supported_transfertools(
+        source_rse: "RseData",
+        dest_rse: "RseData",
+        transfertools: "Optional[list[str]]" = None,
+        *,
+        session: "Session",
+) -> set[str]:
+
+    if not transfertools:
+        transfertools = list(TRANSFERTOOL_CLASSES_BY_NAME)
+
+    source_rse.ensure_loaded(load_attributes=True, session=session)
+    dest_rse.ensure_loaded(load_attributes=True, session=session)
+
+    result = set()
+    for tt_name in transfertools:
+        tt_class = TRANSFERTOOL_CLASSES_BY_NAME.get(tt_name)
+        if tt_class and tt_class.can_perform_transfer(source_rse, dest_rse):
+            result.add(tt_name)
+    return result