rucio 32.5.1__tar.gz → 32.6.0.post1__tar.gz

{rucio-32.5.1 → rucio-32.6.0.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 32.5.1
+Version: 32.6.0.post1
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/api/replica.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import datetime
-import uuid
 from typing import TYPE_CHECKING
 
 from rucio.api import permission
@@ -85,21 +84,23 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
     if not replicas:
         return {}
 
-    kwargs = {}
-    rse_map = {}                # RSE name -> RSE id
-    if not permission.has_permission(issuer=issuer, vo=vo, action='declare_bad_file_replicas', kwargs=kwargs, session=session):
-        raise exception.AccessDenied('Account %s can not declare bad replicas' % (issuer))
-
-    issuer = InternalAccount(issuer, vo=vo)
+    as_pfns = isinstance(replicas[0], str)
 
     # make sure all elements are either strings or dicts, without mixing
-    type_ = type(replicas[0])
-    if any(not isinstance(r, type_) for r in replicas):
+    if any(isinstance(r, str) != as_pfns for r in replicas):
         raise exception.InvalidType('The replicas must be specified either as a list of PFNs (strings) or list of dicts')
 
+    rse_map = {}                # RSE name -> RSE id
+
     replicas_lst = replicas
-    if type_ is dict:
-        replicas_lst = []           # need to create new list to convert replica["scope"] from strings to InternalScope objects
+    rse_ids_to_check = set()        # to check for permission to declare bad replicas
+    if as_pfns:
+        scheme, rses_for_replicas, unknowns = replica.get_pfn_to_rse(replicas, vo=vo, session=session)
+        if unknowns:
+            raise exception.ReplicaNotFound("Not all replicas found")
+        rse_ids_to_check = set(rses_for_replicas.keys())
+    else:
+        replicas_lst = []
         for r in replicas:
             if "name" not in r or "scope" not in r or ("rse" not in r and "rse_id" not in r):
                 raise exception.InvalidType('The replica dictionary must include scope and either rse (name) or rse_id')
@@ -109,14 +110,24 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
                 rse = r["rse"]
                 rse_map[rse] = rse_id = get_rse_id(rse=rse, vo=vo, session=session)
             replicas_lst.append({
-                "scope": scope,
                 "rse_id": rse_id,
+                "scope": scope,
                 "name": r["name"]
             })
+            rse_ids_to_check.add(rse_id)
 
     rse_id_to_name = invert_dict(rse_map)   # RSE id -> RSE name
 
-    undeclared = replica.declare_bad_file_replicas(replicas_lst, reason=reason, issuer=issuer, status=BadFilesStatus.BAD,
+    for rse_id in rse_ids_to_check:
+        if not permission.has_permission(issuer=issuer, vo=vo, action='declare_bad_file_replicas',
+                                         kwargs={"rse_id": rse_id},
+                                         session=session):
+            raise exception.AccessDenied('Account %s can not declare bad replicas in RSE %s' %
+                                         (issuer, rse_id_to_name.get(rse_id, rse_id)))
+
+    undeclared = replica.declare_bad_file_replicas(replicas_lst, reason=reason,
+                                                   issuer=InternalAccount(issuer, vo=vo),
+                                                   status=BadFilesStatus.BAD,
                                                    force=force, session=session)
     out = {}
     for rse_id, ulist in undeclared.items():
@@ -128,7 +139,6 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
                 rse_name = rse_id_to_name[rse_id]
             else:
                 try:
-                    uuid.UUID(rse_id)
                     rse_name = get_rse_name(rse_id=rse_id, session=session)
                 except (ValueError, exception.RSENotFound):
                     rse_name = str(rse_id)

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/common/config.py

@@ -347,7 +347,7 @@ def config_get_int(
         default=default,
         check_config_table=check_config_table,
         session=session,
-        use_cache=True,
+        use_cache=use_cache,
         expiration_time=expiration_time,
         convert_type_fnc=int,
     )
@@ -434,7 +434,7 @@ def config_get_float(
         default=default,
         check_config_table=check_config_table,
         session=session,
-        use_cache=True,
+        use_cache=use_cache,
         expiration_time=expiration_time,
         convert_type_fnc=float,
     )
@@ -521,7 +521,7 @@ def config_get_bool(
         default=default,
         check_config_table=check_config_table,
         session=session,
-        use_cache=True,
+        use_cache=use_cache,
         expiration_time=expiration_time,
         convert_type_fnc=_convert_to_boolean,
     )
@@ -608,7 +608,7 @@ def config_get_list(
         default=default,
         check_config_table=check_config_table,
         session=session,
-        use_cache=True,
+        use_cache=use_cache,
         expiration_time=expiration_time,
     )
     if isinstance(value, str):

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/did.py

@@ -1655,20 +1655,22 @@ def list_content(scope, name, *, session: "Session"):
     :param name: The data identifier name.
     :param session: The database session in use.
     """
-    try:
-        stmt = select(
-            models.DataIdentifierAssociation
-        ).with_hint(
-            models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle'
-        ).filter_by(
-            scope=scope,
-            name=name
-        )
-        for tmp_did in session.execute(stmt).yield_per(5).scalars():
-            yield {'scope': tmp_did.child_scope, 'name': tmp_did.child_name, 'type': tmp_did.child_type,
-                   'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5}
-    except NoResultFound:
-        raise exception.DataIdentifierNotFound(f"Data identifier '{scope}:{name}' not found")
+    stmt = select(
+        models.DataIdentifierAssociation
+    ).with_hint(
+        models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle'
+    ).filter_by(
+        scope=scope,
+        name=name
+    )
+    children_found = False
+    for tmp_did in session.execute(stmt).yield_per(5).scalars():
+        children_found = True
+        yield {'scope': tmp_did.child_scope, 'name': tmp_did.child_name, 'type': tmp_did.child_type,
+               'bytes': tmp_did.bytes, 'adler32': tmp_did.adler32, 'md5': tmp_did.md5}
+    if not children_found:
+        # Raise exception if the did doesn't exist
+        __get_did(scope=scope, name=name, session=session)
 
 
 @stream_session

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/did_meta_plugins/filter_engine.py

@@ -265,7 +265,9 @@ class FilterEngine:
             else:
                 return value
         try:
-            value = ast.literal_eval(value)     # will catch float, int and bool
+            operators = ('+', '-', '*', '/')
+            if not any(operator in value for operator in operators):    # fix for lax ast literal_eval in earlier python versions
+                value = ast.literal_eval(value)                         # will catch float, int and bool
         except (ValueError, SyntaxError):
             pass
         return value

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/request.py

@@ -621,16 +621,17 @@ def fetch_paths(request_id, *, session: "Session"):
 @METRICS.time_it
 @transactional_session
 def get_and_mark_next(
+        rse_collection: "RseCollection",
         request_type,
         state,
         processed_by: Optional[str] = None,
         processed_at_delay: int = 600,
-        limit=100,
-        older_than=None,
-        rse_id=None,
-        activity=None,
-        total_workers=0,
-        worker_number=0,
+        limit: int = 100,
+        older_than: "Optional[datetime.datetime]" = None,
+        rse_id: "Optional[str]" = None,
+        activity: "Optional[str]" = None,
+        total_workers: int = 0,
+        worker_number: int = 0,
         mode_all=False,
         hash_variable='id',
         activity_shares=None,
@@ -643,6 +644,7 @@ def get_and_mark_next(
     Retrieve the next requests matching the request type and state.
     Workers are balanced via hashing to reduce concurrency on database.
 
+    :param rse_collection:    the RSE collection being used
     :param request_type:      Type of the request as a string or list of strings.
     :param state:             State of the request as a string or list of strings.
     :param processed_by:      the daemon/executable running this query
@@ -682,7 +684,7 @@ def get_and_mark_next(
     for share in activity_shares:
 
         query = select(
-            models.Request
+            models.Request.id
         ).where(
             models.Request.state.in_(state),
             models.Request.request_type.in_(request_type)
@@ -738,6 +740,21 @@ def get_and_mark_next(
         else:
             query = query.limit(limit)
 
+        if session.bind.dialect.name == 'oracle':
+            query = select(
+                models.Request
+            ).where(
+                models.Request.id.in_(query)
+            ).with_for_update(
+                skip_locked=True
+            )
+        else:
+            query = query.with_only_columns(
+                models.Request
+            ).with_for_update(
+                skip_locked=True,
+                of=models.Request.last_processed_by
+            )
         query_result = session.execute(query).scalars()
         if query_result:
             if mode_all:
@@ -748,8 +765,8 @@ def get_and_mark_next(
 
                     dst_id = res_dict['dest_rse_id']
                     src_id = res_dict['source_rse_id']
-                    res_dict['dest_rse'] = get_rse_name(rse_id=dst_id, session=session) if dst_id is not None else None
-                    res_dict['source_rse'] = get_rse_name(rse_id=src_id, session=session) if src_id is not None else None
+                    res_dict['dst_rse'] = rse_collection[dst_id].ensure_loaded(load_name=True)
+                    res_dict['src_rse'] = rse_collection[src_id].ensure_loaded(load_name=True) if src_id is not None else None
 
                     result.append(res_dict)
             else:
@@ -2018,46 +2035,6 @@ def update_requests_priority(priority, filter_, *, session: "Session", logger=lo
         raise RucioException(error.args)
 
 
-@transactional_session
-def update_request_state(tt_status_report, *, session: "Session", logger=logging.log):
-    """
-    Used by poller and consumer to update the internal state of requests,
-    after the response by the external transfertool.
-
-    :param tt_status_report:      The transfertool status update, retrieved via request.query_request().
-    :param session:               The database session to use.
-    :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
-    :returns commit_or_rollback:  Boolean.
-    """
-
-    request_id = tt_status_report.request_id
-    try:
-        fields_to_update = tt_status_report.get_db_fields_to_update(session=session, logger=logger)
-        if not fields_to_update:
-            update_request(request_id, raise_on_missing=True, session=session)
-            return False
-        else:
-            logger(logging.INFO, 'UPDATING REQUEST %s FOR %s with changes: %s' % (str(request_id), tt_status_report, fields_to_update))
-
-            set_request_state(request_id, session=session, **fields_to_update)
-            request = tt_status_report.request(session)
-
-            if tt_status_report.state == RequestState.FAILED:
-                if is_intermediate_hop(request):
-                    handle_failed_intermediate_hop(request, session=session)
-
-            add_monitor_message(new_state=tt_status_report.state,
-                                request=request,
-                                additional_fields=tt_status_report.get_monitor_msg_fields(session=session, logger=logger),
-                                session=session)
-            return True
-    except UnsupportedOperation as error:
-        logger(logging.WARNING, "Request %s doesn't exist - Error: %s" % (request_id, str(error).replace('\n', '')))
-        return False
-    except Exception:
-        logger(logging.CRITICAL, "Exception", exc_info=True)
-
-
 @read_session
 def add_monitor_message(new_state, request, additional_fields, *, session: "Session"):
     """

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/rse.py

@@ -29,7 +29,7 @@ from sqlalchemy.sql.expression import or_, and_, desc, true, false, func, select
 
 from rucio.common import exception, utils
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import get_lfn2pfn_algorithm_default, config_get_bool
+from rucio.common.config import get_lfn2pfn_algorithm_default
 from rucio.common import types
 from rucio.common.utils import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS, Availability
 from rucio.core.rse_counter import add_counter, get_counter
@@ -159,8 +159,7 @@ class RseData:
         Given a dict of RseData objects indexed by rse_id, ensure that the desired fields are initialised
         in all objects from the input.
         """
-        use_temp_tables = config_get_bool('core', 'use_temp_tables', default=True, session=session)
-        if not use_temp_tables or len(rse_id_to_data) < 4:
+        if len(rse_id_to_data) < 4:  # 4 was selected without particular reason as "seems good enough"
             for rse_data in rse_id_to_data.values():
                 rse_data.ensure_loaded(
                     load_name=load_name,
@@ -224,7 +223,7 @@ class RseData:
                                                                   session=session))
             for rse_id, db_rse in db_rses_by_id.items():
                 rse_data = rse_id_to_data[rse_id]
-                settings = rse_data._attributes if rse_data._attributes is not None else settings_by_id[rse_id]
+                settings = rse_data._attributes if rse_data._attributes is not None else settings_by_id.get(rse_id, {})
                 columns = _format_get_rse(db_rse=db_rse, rse_attributes=settings, session=session)
                 rse_data._columns = columns
                 rse_data._name = columns['rse']
@@ -281,8 +280,8 @@ class RseData:
         if load_name:
             # The name could have been loaded already (when loading columns or info). Skip loading if it's known.
             if not load_columns and not load_info:
-                for rse_id, rse_data in rse_id_to_data.items():
-                    rse_data._name = db_rses_by_id[rse_id].rse
+                for rse_id in rse_ids_to_load:
+                    rse_id_to_data[rse_id]._name = db_rses_by_id[rse_id].rse
 
 
 class RseCollection(Generic[T]):

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/topology.py

@@ -18,6 +18,7 @@ import logging
 import threading
 import weakref
 from collections.abc import Callable, Iterable, Iterator
+from decimal import Decimal
 from typing import TYPE_CHECKING, cast, Any, Generic, Optional, TypeVar, Union
 
 from sqlalchemy import and_, select
@@ -32,7 +33,7 @@ from rucio.db.sqla.session import read_session, transactional_session
 from rucio.rse import rsemanager as rsemgr
 
 LoggerFunction = Callable[..., Any]
-_Number = Union[float, int]
+_Number = Union[int, Decimal]
 TN = TypeVar("TN", bound="Node")
 TE = TypeVar("TE", bound="Edge")
 
@@ -50,6 +51,9 @@ if TYPE_CHECKING:
         def enabled(self) -> bool:
             ...
 
+    TNState = TypeVar("TNState", bound=_StateProvider)
+    TEState = TypeVar("TEState", bound=_StateProvider)
+
 
 DEFAULT_HOP_PENALTY = 10
 INF = float('inf')
@@ -344,9 +348,9 @@ class Topology(RseCollection, Generic[TN, TE]):
             self,
             dst_node: TN,
             nodes_to_find: Optional[set[TN]] = None,
-            node_state_provider: "Callable[[TN], _StateProvider]" = lambda x: x,
-            edge_state_provider: "Callable[[TE], _StateProvider]" = lambda x: x,
-    ) -> "Iterator[tuple[TN, _Number, _StateProvider, TE, _StateProvider]]":
+            node_state_provider: "Callable[[TN], TNState]" = lambda x: x,
+            edge_state_provider: "Callable[[TE], TEState]" = lambda x: x,
+    ) -> "Iterator[tuple[TN, _Number, TNState, TE, TEState]]":
         """
         Does a Backwards Dijkstra's algorithm: start from destination and follow inbound links to other nodes.
         If multihop is disabled, stop after analysing direct connections to dest_rse.
@@ -358,7 +362,7 @@ class Topology(RseCollection, Generic[TN, TE]):
 
         priority_q = PriorityQueue()
         priority_q[dst_node] = 0
-        next_hops: dict[TN, tuple[_Number, _StateProvider, Optional[TE], Optional[_StateProvider]]] =\
+        next_hops: dict[TN, tuple[_Number, TNState, Optional[TE], Optional[TEState]]] =\
             {dst_node: (0, node_state_provider(dst_node), None, None)}
         while priority_q:
             node = priority_q.pop()

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/core/transfer.py

@@ -30,7 +30,7 @@ from rucio.common.config import config_get
 from rucio.common.constants import SUPPORTED_PROTOCOLS
 from rucio.common.exception import (InvalidRSEExpression,
                                     RequestNotFound, RSEProtocolNotSupported,
-                                    RucioException)
+                                    RucioException, UnsupportedOperation)
 from rucio.common.utils import construct_surl
 from rucio.core import did, message as message_core, request as request_core
 from rucio.core.account import list_accounts
@@ -42,6 +42,7 @@ from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection
 from rucio.db.sqla.session import read_session, transactional_session, stream_session
 from rucio.rse import rsemanager as rsemgr
+from rucio.transfertool.transfertool import TransferStatusReport
 from rucio.transfertool.fts3 import FTS3Transfertool
 from rucio.transfertool.globus import GlobusTransferTool
 from rucio.transfertool.mock import MockTransfertool
@@ -500,6 +501,48 @@ def set_transfers_state(
     logger(logging.DEBUG, 'Finished to register transfer state for %s' % external_id)
 
 
+@transactional_session
+def update_transfer_state(tt_status_report: TransferStatusReport, *, session: "Session", logger=logging.log):
+    """
+    Used by poller and consumer to update the internal state of requests,
+    after the response by the external transfertool.
+
+    :param tt_status_report:      The transfertool status update, retrieved via request.query_request().
+    :param session:               The database session to use.
+    :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
+    :returns commit_or_rollback:  Boolean.
+    """
+
+    request_id = tt_status_report.request_id
+    try:
+        fields_to_update = tt_status_report.get_db_fields_to_update(session=session, logger=logger)
+        if not fields_to_update:
+            request_core.update_request(request_id, raise_on_missing=True, session=session)
+            return False
+        else:
+            logger(logging.INFO, 'UPDATING REQUEST %s FOR %s with changes: %s' % (str(request_id), tt_status_report, fields_to_update))
+
+            set_request_state(request_id, session=session, **fields_to_update)
+            request = tt_status_report.request(session)
+
+            if tt_status_report.state == RequestState.FAILED:
+                if request_core.is_intermediate_hop(request):
+                    request_core.handle_failed_intermediate_hop(request, session=session)
+
+            request_core.add_monitor_message(
+                new_state=tt_status_report.state,
+                request=request,
+                additional_fields=tt_status_report.get_monitor_msg_fields(session=session, logger=logger),
+                session=session
+            )
+            return True
+    except UnsupportedOperation as error:
+        logger(logging.WARNING, "Request %s doesn't exist - Error: %s" % (request_id, str(error).replace('\n', '')))
+        return False
+    except Exception:
+        logger(logging.CRITICAL, "Exception", exc_info=True)
+
+
 @transactional_session
 def mark_transfer_lost(request, *, session: "Session", logger=logging.log):
     new_state = RequestState.LOST

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/daemons/conveyor/finisher.py

@@ -41,10 +41,11 @@ from rucio.common.utils import chunks
 from rucio.core import request as request_core, replica as replica_core
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import list_rses
+from rucio.core.transfer import ProtocolFactory
+from rucio.core.topology import Topology, ExpiringObjectCache
 from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
 from rucio.db.sqla.constants import RequestState, RequestType, ReplicaState, BadFilesStatus
 from rucio.db.sqla.session import transactional_session
-from rucio.rse import rsemanager
 
 if TYPE_CHECKING:
     from rucio.daemons.common import HeartbeatHandler
@@ -60,6 +61,7 @@ FAILED_DURING_SUBMISSION_DELAY = datetime.timedelta(minutes=120)
 def _fetch_requests(
         db_bulk,
         set_last_processed_by: bool,
+        cached_topology,
         heartbeat_handler,
         activity,
 ):
@@ -67,8 +69,11 @@ def _fetch_requests(
 
     logger(logging.DEBUG, 'Working on activity %s', activity)
 
+    topology = cached_topology.get() if cached_topology else Topology()
+
     get_requests_fnc = functools.partial(
         request_core.get_and_mark_next,
+        rse_collection=topology,
         request_type=[RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT],
         processed_by=heartbeat_handler.short_executable if set_last_processed_by else None,
         limit=db_bulk,
@@ -100,17 +105,18 @@ def _fetch_requests(
     if len(reqs) < db_bulk / 2:
         logger(logging.INFO, "Only %s transfers, which is less than half of the bulk %s", len(reqs), db_bulk)
         must_sleep = True
-    return must_sleep, reqs
+    return must_sleep, (reqs, topology)
 
 
 def _handle_requests(
-        reqs,
+        batch,
         bulk,
         suspicious_patterns,
         retry_protocol_mismatches,
         *,
         logger=logging.log,
 ):
+    reqs, topology = batch
     if not reqs:
         return
 
@@ -122,7 +128,7 @@ def _handle_requests(
         for chunk in chunks(reqs, bulk):
             try:
                 stopwatch = Stopwatch()
-                _finish_requests(chunk, suspicious_patterns, retry_protocol_mismatches, logger=logger)
+                _finish_requests(topology, chunk, suspicious_patterns, retry_protocol_mismatches, logger=logger)
                 METRICS.timer('handle_requests_time').observe(stopwatch.elapsed / (len(chunk) or 1))
                 METRICS.counter('handle_requests').inc(len(chunk))
             except Exception as error:
@@ -144,6 +150,7 @@ def finisher(
         bulk=100,
         db_bulk=1000,
         partition_wait_time=10,
+        cached_topology=None,
         total_threads=1,
 ):
     """
@@ -172,14 +179,15 @@ def finisher(
     def _db_producer(*, activity: str, heartbeat_handler: "HeartbeatHandler"):
         return _fetch_requests(
             db_bulk=db_bulk,
+            cached_topology=cached_topology,
             activity=activity,
             set_last_processed_by=not once,
             heartbeat_handler=heartbeat_handler,
         )
 
-    def _consumer(reqs):
+    def _consumer(batch):
         return _handle_requests(
-            reqs=reqs,
+            batch=batch,
             bulk=bulk,
             suspicious_patterns=suspicious_patterns,
             retry_protocol_mismatches=retry_protocol_mismatches,
@@ -209,17 +217,19 @@ def run(once=False, total_threads=1, sleep_time=60, activities=None, bulk=100, d
     if rucio.db.sqla.util.is_old_db():
         raise DatabaseException('Database was not updated, daemon won\'t start')
 
+    cached_topology = ExpiringObjectCache(ttl=300, new_obj_fnc=lambda: Topology())
     finisher(
         once=once,
         activities=activities,
         bulk=bulk,
         db_bulk=db_bulk,
         sleep_time=sleep_time,
+        cached_topology=cached_topology,
         total_threads=total_threads
     )
 
 
-def _finish_requests(reqs, suspicious_patterns, retry_protocol_mismatches, logger=logging.log):
+def _finish_requests(topology: "Topology", reqs, suspicious_patterns, retry_protocol_mismatches, logger=logging.log):
     """
     Used by finisher to handle terminated requests,
 
@@ -231,7 +241,7 @@ def _finish_requests(reqs, suspicious_patterns, retry_protocol_mismatches, logge
     failed_during_submission = [RequestState.SUBMITTING, RequestState.SUBMISSION_FAILED, RequestState.LOST]
     failed_no_submission_attempts = [RequestState.NO_SOURCES, RequestState.ONLY_TAPE_SOURCES, RequestState.MISMATCH_SCHEME]
     undeterministic_rses = __get_undeterministic_rses(logger=logger)
-    rses_info, protocols = {}, {}
+    protocol_factory = ProtocolFactory()
     replicas = {}
     for req in reqs:
         try:
@@ -252,14 +262,11 @@ def _finish_requests(reqs, suspicious_patterns, retry_protocol_mismatches, logge
 
                 # for TAPE, replica path is needed
                 if req['request_type'] in (RequestType.TRANSFER, RequestType.STAGEIN) and req['dest_rse_id'] in undeterministic_rses:
-                    if req['dest_rse_id'] not in rses_info:
-                        rses_info[req['dest_rse_id']] = rsemanager.get_rse_info(rse_id=req['dest_rse_id'])
+                    dst_rse = topology[req['dest_rse_id']].ensure_loaded(load_info=True)
                     pfn = req['dest_url']
                     scheme = urlparse(pfn).scheme
-                    dest_rse_id_scheme = '%s_%s' % (req['dest_rse_id'], scheme)
-                    if dest_rse_id_scheme not in protocols:
-                        protocols[dest_rse_id_scheme] = rsemanager.create_protocol(rses_info[req['dest_rse_id']], 'write', scheme)
-                    path = protocols[dest_rse_id_scheme].parse_pfns([pfn])[pfn]['path']
+                    protocol = protocol_factory.protocol(dst_rse, scheme, 'write')
+                    path = protocol.parse_pfns([pfn])[pfn]['path']
                     replica['path'] = os.path.join(path, os.path.basename(pfn))
 
                 # replica should not be added to replicas until all info are filled

{rucio-32.5.1 → rucio-32.6.0.post1}/lib/rucio/daemons/conveyor/poller.py

@@ -40,6 +40,7 @@ from rucio.common.types import InternalAccount
 from rucio.common.utils import dict_chunks
 from rucio.core import transfer as transfer_core, request as request_core
 from rucio.core.monitor import MetricManager
+from rucio.core.topology import Topology, ExpiringObjectCache
 from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
 from rucio.db.sqla.constants import RequestState, RequestType
 from rucio.transfertool.fts3 import FTS3Transfertool
@@ -63,15 +64,21 @@ def _fetch_requests(
         activity_shares,
         transfertool,
         filter_transfertool,
+        cached_topology,
         activity,
+        set_last_processed_by: bool,
         heartbeat_handler
 ):
     worker_number, total_workers, logger = heartbeat_handler.live()
 
     logger(logging.DEBUG, 'Start to poll transfers older than %i seconds for activity %s using transfer tool: %s' % (older_than, activity, filter_transfertool))
+
+    topology = cached_topology.get() if cached_topology else Topology()
     transfs = request_core.get_and_mark_next(
+        rse_collection=topology,
         request_type=[RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT],
         state=[RequestState.SUBMITTED],
+        processed_by=heartbeat_handler.short_executable if set_last_processed_by else None,
         limit=db_bulk,
         older_than=datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than) if older_than else None,
         total_workers=total_workers,
@@ -151,6 +158,7 @@ def poller(
         partition_wait_time: int = 10,
         transfertool: Optional[str] = TRANSFER_TOOL,
         filter_transfertool: Optional[str] = FILTER_TRANSFERTOOL,
+        cached_topology=None,
         total_threads: int = 1,
 ):
     """
@@ -189,7 +197,9 @@ def poller(
             activity_shares=activity_shares,
             transfertool=transfertool,
             filter_transfertool=filter_transfertool,
+            cached_topology=cached_topology,
             activity=activity,
+            set_last_processed_by=not once,
             heartbeat_handler=heartbeat_handler,
         )
 
@@ -256,6 +266,7 @@ def run(
         parsed_activity_shares.update((share, int(percentage * db_bulk)) for share, percentage in parsed_activity_shares.items())
         logging.info('activity shares enabled: %s' % parsed_activity_shares)
 
+    cached_topology = ExpiringObjectCache(ttl=300, new_obj_fnc=lambda: Topology())
     poller(
         once=once,
         fts_bulk=fts_bulk,
@@ -264,6 +275,7 @@ def run(
         sleep_time=sleep_time,
         activities=activities,
         activity_shares=parsed_activity_shares,
+        cached_topology=cached_topology,
         total_threads=total_threads,
     )
 
@@ -343,7 +355,7 @@ def _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger):
                 METRICS.counter('query_transfer_exception').inc()
             else:
                 for request_id in request_ids.intersection(transf_resp):
-                    ret = request_core.update_request_state(transf_resp[request_id], logger=logger)
+                    ret = transfer_core.update_transfer_state(transf_resp[request_id], logger=logger)
                     # if True, really update request content; if False, only touch request
                     if ret:
                         cnt += 1