rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/transfertool/fts3.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,10 +18,9 @@ import logging
 import pathlib
 import traceback
 import uuid
-from collections.abc import Callable
 from configparser import NoOptionError, NoSectionError
 from json import loads
-from typing import Any, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, Union
 from urllib.parse import urlparse
 
 import requests
@@ -31,20 +29,26 @@ from requests.adapters import ReadTimeout
 from requests.packages.urllib3 import disable_warnings  # pylint: disable=import-error
 
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get, config_get_bool, config_get_int
-from rucio.common.constants import FTS_JOB_TYPE, FTS_STATE, FTS_COMPLETE_STATE
-from rucio.common.exception import TransferToolTimeout, TransferToolWrongAnswer, DuplicateFileTransferSubmission
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
+from rucio.common.constants import FTS_COMPLETE_STATE, FTS_JOB_TYPE, FTS_STATE, RseAttr
+from rucio.common.exception import DuplicateFileTransferSubmission, TransferToolTimeout, TransferToolWrongAnswer
 from rucio.common.stopwatch import Stopwatch
-from rucio.common.utils import APIEncoder, chunks, PREFERRED_CHECKSUM
+from rucio.common.utils import PREFERRED_CHECKSUM, APIEncoder, chunks, deep_merge_dict
 from rucio.core.monitor import MetricManager
-from rucio.core.oidc import get_token_for_account_operation
+from rucio.core.oidc import request_token
 from rucio.core.request import get_source_rse, get_transfer_error
-from rucio.core.rse import get_rse_supported_checksums_from_attributes
+from rucio.core.rse import determine_audience_for_rse, determine_scope_for_rse, get_rse_supported_checksums_from_attributes
 from rucio.db.sqla.constants import RequestState
-from rucio.transfertool.transfertool import Transfertool, TransferToolBuilder, TransferStatusReport
+from rucio.transfertool.fts3_plugins import FTS3TapeMetadataPlugin
+from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder
 
 if TYPE_CHECKING:
-    from rucio.core.transfer import DirectTransferDefinition
+    from collections.abc import Iterable, Sequence
+
+    from sqlalchemy.orm import Session
+
+    from rucio.common.types import LoggerFunction
+    from rucio.core.request import DirectTransfer
     from rucio.core.rse import RseData
 
 logging.getLogger("requests").setLevel(logging.CRITICAL)
@@ -70,9 +74,6 @@ BULK_QUERY_COUNTER = METRICS.counter(name='{host}.bulk_query.{state}',
 QUERY_DETAILS_COUNTER = METRICS.counter(name='{host}.query_details.{state}',
                                         documentation='Number of detailed status queries', labelnames=('state', 'host'))
 
-ALLOW_USER_OIDC_TOKENS = config_get_bool('conveyor', 'allow_user_oidc_tokens', False, False)
-REQUEST_OIDC_SCOPE = config_get('conveyor', 'request_oidc_scope', False, 'fts:submit-transfer')
-REQUEST_OIDC_AUDIENCE = config_get('conveyor', 'request_oidc_audience', False, 'fts:example')
 REWRITE_HTTPS_TO_DAVS = config_get_bool('transfers', 'rewrite_https_to_davs', default=False)
 VO_CERTS_PATH = config_get('conveyor', 'vo_certs_path', False, None)
 
@@ -109,7 +110,7 @@ _SCITAGS_EXP_ID = None
 _SCITAGS_ACTIVITY_IDS = {}
 
 
-def _scitags_ids(logger: Callable[..., Any] = logging.log) -> "tuple[int | None, dict[str, int]]":
+def _scitags_ids(logger: "LoggerFunction" = logging.log) -> "tuple[int | None, dict[str, int]]":
     """
     Re-fetch if needed and return the scitags ids
     """
@@ -124,7 +125,7 @@ def _scitags_ids(logger: Callable[..., Any] = logging.log) -> "tuple[int | None,
     if _SCITAGS_NEXT_REFRESH < now:
         exp_name = config_get('packet-marking', 'exp_name', default='')
         fetch_url = config_get('packet-marking', 'fetch_url', default='https://www.scitags.org/api.json')
-        fetch_interval = config_get_int('packet-marking', 'fetch_interval', default=datetime.timedelta(hours=48).seconds)
+        fetch_interval = config_get_int('packet-marking', 'fetch_interval', default=int(datetime.timedelta(hours=48).total_seconds()))
         fetch_timeout = config_get_int('packet-marking', 'fetch_timeout', default=5)
 
         _SCITAGS_NEXT_REFRESH = now + datetime.timedelta(seconds=fetch_interval)
@@ -160,7 +161,7 @@ def _scitags_ids(logger: Callable[..., Any] = logging.log) -> "tuple[int | None,
     return _SCITAGS_EXP_ID, _SCITAGS_ACTIVITY_IDS
 
 
-def _pick_cert_file(vo: "Optional[str]") -> "Optional[str]":
+def _pick_cert_file(vo: Optional[str]) -> Optional[str]:
     cert = None
     if vo:
         vo_cert = config_get('vo_certs', vo, False, None)
@@ -177,7 +178,7 @@ def _pick_cert_file(vo: "Optional[str]") -> "Optional[str]":
     return cert
 
 
-def _configured_source_strategy(activity: str, logger: Callable[..., Any]) -> str:
+def _configured_source_strategy(activity: str, logger: "LoggerFunction") -> str:
     """
     Retrieve from the configuration the source selection strategy for the given activity
     """
@@ -198,36 +199,20 @@ def _configured_source_strategy(activity: str, logger: Callable[..., Any]) -> st
     return activity_source_strategy.get(str(activity), default_source_strategy)
 
 
-def oidc_supported(transfer_hop) -> bool:
-    """
-    checking OIDC AuthN/Z support per destination and source RSEs;
-
-    for oidc_support to be activated, all sources and the destination must explicitly support it
-    """
-    # assumes use of boolean 'oidc_support' RSE attribute
-    if not transfer_hop.dst.rse.attributes.get('oidc_support', False):
-        return False
-
-    for source in transfer_hop.sources:
-        if not source.rse.attributes.get('oidc_support', False):
-            return False
-    return True
-
-
 def _available_checksums(
-        transfer: "DirectTransferDefinition",
+        transfer: "DirectTransfer",
 ) -> tuple[set[str], set[str]]:
     """
     Get checksums which can be used for file validation on the source and the destination RSE
     """
     src_attributes = transfer.src.rse.attributes
-    if src_attributes.get('verify_checksum', True):
+    if src_attributes.get(RseAttr.VERIFY_CHECKSUM, True):
         src_checksums = set(get_rse_supported_checksums_from_attributes(src_attributes))
     else:
         src_checksums = set()
 
     dst_attributes = transfer.dst.rse.attributes
-    if dst_attributes.get('verify_checksum', True):
+    if dst_attributes.get(RseAttr.VERIFY_CHECKSUM, True):
         dst_checksums = set(get_rse_supported_checksums_from_attributes(dst_attributes))
     else:
         dst_checksums = set()
@@ -236,8 +221,8 @@ def _available_checksums(
 
 
 def _hop_checksum_validation_strategy(
-        transfer: "DirectTransferDefinition",
-        logger: Callable[..., Any],
+        transfer: "DirectTransfer",
+        logger: "LoggerFunction",
 ) -> tuple[str, set[str]]:
     """
     Compute the checksum validation strategy (none, source, destination or both) depending
@@ -262,8 +247,8 @@ def _hop_checksum_validation_strategy(
 
 
 def _path_checksum_validation_strategy(
-        transfer_path: "list[DirectTransferDefinition]",
-        logger: Callable[..., Any],
+        transfer_path: "list[DirectTransfer]",
+        logger: "LoggerFunction",
 ) -> str:
     """
     Compute the checksum validation strategy for the whole transfer path.
@@ -279,7 +264,7 @@ def _path_checksum_validation_strategy(
 
 
 def _pick_fts_checksum(
-        transfer: "DirectTransferDefinition",
+        transfer: "DirectTransfer",
         path_strategy: "str",
 ) -> Optional[str]:
     """
@@ -313,9 +298,31 @@ def _pick_fts_checksum(
     return checksum_to_use
 
 
-def build_job_params(transfer_path, bring_online, default_lifetime, archive_timeout_override, max_time_in_queue, logger):
+def _use_tokens(transfer_hop: "DirectTransfer"):
+    """Whether a transfer can be performed with tokens.
+
+    In order to be so, all the involved RSEs must have it explicitly enabled
+    and the protocol being used must be WebDAV.
+    """
+    for endpoint in [*transfer_hop.sources, transfer_hop.dst]:
+        if (endpoint.rse.attributes.get(RseAttr.OIDC_SUPPORT) is not True
+                or endpoint.scheme != 'davs'):
+            return False
+    return True
+
+
+def build_job_params(
+        transfer_path: list["DirectTransfer"],
+        bring_online: Optional[int] = None,
+        default_lifetime: Optional[int] = None,
+        archive_timeout_override: Optional[int] = None,
+        max_time_in_queue: Optional[dict] = None,
+        logger: "LoggerFunction" = logging.log
+) -> dict[str, Any]:
     """
     Prepare the job parameters which will be passed to FTS transfertool
+    Please refer to https://fts3-docs.web.cern.ch/fts3-docs/fts-rest/docs/bulk.html#parameters
+    for the list of parameters.
     """
 
     # The last hop is the main request (the one which triggered the whole transfer),
@@ -323,13 +330,46 @@ def build_job_params(transfer_path, bring_online, default_lifetime, archive_time
     last_hop = transfer_path[-1]
     first_hop = transfer_path[0]
 
-    overwrite, bring_online_local = True, None
+    # Overwriting by default is set to True for non TAPE RSEs.
+    # Tape RSEs can force overwrite by setting the "overwrite" attribute to True.
+    # There is yet another configuration option: transfers->overwrite_corrupted_files that when is set to True
+    # it will retry failed requests with overwrite flag set to True
+    overwrite, overwrite_when_only_on_disk, bring_online_local = True, False, None
+
     if first_hop.src.rse.is_tape_or_staging_required():
         # Activate bring_online if it was requested by first hop
         # We don't allow multihop via a tape, so bring_online should not be set on any other hop
         bring_online_local = bring_online
+
     if last_hop.dst.rse.is_tape():
-        overwrite = False
+        # FTS v3.12.12 introduced a new boolean parameter "overwrite_when_only_on_disk" that controls if the file can be overwritten
+        # in TAPE enabled RSEs ONLY IF the file is on the disk buffer and not yet committed to tape media.
+        # This functionality should reduce the number of stuck files in the disk buffer that are not migrated to tape media (for whatever reason).
+        # Please be aware that FTS does not guarantee an atomic operation from the time it checks for existence of the file on disk and tape and
+        # the moment the file is overwritten, so there is a race condition that could overwrite the file on the tape media
+
+        # Setting both flags is incompatible, so we opt in for the safest approach: "overwrite_when_only_on_disk"
+        # this is aligned with FTS implementation: see (internal access only): https://its.cern.ch/jira/browse/FTS-2007
+
+        overwrite_when_only_on_disk = last_hop.dst.rse.attributes.get('overwrite_when_only_on_disk', False)
+        overwrite = False if overwrite_when_only_on_disk else last_hop.dst.rse.attributes.get('overwrite', False)
+
+    # We still need to check for the
+    # "transfers -> overwrite_corrupted_files setting. The logic behind this flag is that
+    # it will update the rws (RequestWithSources) with the "overwrite" attribute set to True
+    # after finding an 'Destination file exists and overwrite is not enabled' error message
+    overwrite_corrupted_files = last_hop.rws.attributes.get('overwrite', False)
+    if overwrite_corrupted_files:
+        overwrite = True  # both for DISK and TAPE
+
+    logger(logging.DEBUG, 'RSE:%s Is it Tape? %s overwrite_when_only_on_disk:%s overwrite:%s overwrite_corrupted_files=%s' % (
+        last_hop.dst.rse.name,
+        last_hop.dst.rse.is_tape(),
+        overwrite_when_only_on_disk,
+        overwrite,
+        overwrite_corrupted_files))
+
+    logger(logging.DEBUG, 'RSE attributes are: %s' % (last_hop.dst.rse.attributes))
 
     # Get dest space token
     dest_protocol = last_hop.protocol_factory.protocol(last_hop.dst.rse, last_hop.dst.scheme, last_hop.operation_dest)
@@ -338,8 +378,8 @@ def build_job_params(transfer_path, bring_online, default_lifetime, archive_time
             dest_protocol.attributes['extended_attributes'] and 'space_token' in dest_protocol.attributes['extended_attributes']:
         dest_spacetoken = dest_protocol.attributes['extended_attributes']['space_token']
 
-    strict_copy = last_hop.dst.rse.attributes.get('strict_copy', False)
-    archive_timeout = last_hop.dst.rse.attributes.get('archive_timeout', None)
+    strict_copy = last_hop.dst.rse.attributes.get(RseAttr.STRICT_COPY, False)
+    archive_timeout = last_hop.dst.rse.attributes.get(RseAttr.ARCHIVE_TIMEOUT, None)
 
     job_params = {'account': last_hop.rws.account,
                   'verify_checksum': _path_checksum_validation_strategy(transfer_path, logger=logger),
@@ -348,29 +388,32 @@ def build_job_params(transfer_path, bring_online, default_lifetime, archive_time
                   'job_metadata': {
                       'issuer': 'rucio',
                       'multi_sources': False,
+                      'overwrite_when_only_on_disk': overwrite_when_only_on_disk,
                   },
-                  'overwrite': last_hop.rws.attributes.get('overwrite', overwrite),
+                  'overwrite': overwrite,
+                  'overwrite_when_only_on_disk': overwrite_when_only_on_disk,
                   'priority': last_hop.rws.priority}
 
     if len(transfer_path) > 1:
         job_params['multihop'] = True
         job_params['job_metadata']['multihop'] = True
-    elif len(last_hop.legacy_sources) > 1:
+    elif len(last_hop.sources) > 1:
         job_params['job_metadata']['multi_sources'] = True
     if strict_copy:
         job_params['strict_copy'] = strict_copy
     if dest_spacetoken:
         job_params['spacetoken'] = dest_spacetoken
-    if last_hop.use_ipv4:
+    if (last_hop.dst.rse.attributes.get(RseAttr.USE_IPV4, False)
+            or any(src.rse.attributes.get(RseAttr.USE_IPV4, False) for src in last_hop.sources)):
         job_params['ipv4'] = True
         job_params['ipv6'] = False
 
     # assume s3alternate True (path-style URL S3 RSEs)
     job_params['s3alternate'] = True
-    src_rse_s3_url_style = first_hop.src.rse.attributes.get('s3_url_style', None)
+    src_rse_s3_url_style = first_hop.src.rse.attributes.get(RseAttr.S3_URL_STYLE, None)
     if src_rse_s3_url_style == "host":
         job_params['s3alternate'] = False
-    dst_rse_s3_url_style = last_hop.dst.rse.attributes.get('s3_url_style', None)
+    dst_rse_s3_url_style = last_hop.dst.rse.attributes.get(RseAttr.S3_URL_STYLE, None)
     if dst_rse_s3_url_style == "host":
         job_params['s3alternate'] = False
 
@@ -393,23 +436,42 @@ def build_job_params(transfer_path, bring_online, default_lifetime, archive_time
         elif 'default' in max_time_in_queue:
             job_params['max_time_in_queue'] = max_time_in_queue['default']
 
-    overwrite_hop = True
-    for transfer_hop in transfer_path[:-1]:
-        # Only allow overwrite if all hops in multihop allow it
-        h_overwrite = transfer_hop.rws.attributes.get('overwrite', True)
-        job_params['overwrite'] = h_overwrite and job_params['overwrite']
-        # Allow overwrite_hop if all intermediate hops allow it (ignoring the last hop)
-        overwrite_hop = h_overwrite and overwrite_hop
+    # Refer to https://its.cern.ch/jira/browse/FTS-1749 for full details (login needed), extract below:
+    # Why the overwrite_hop parameter is needed?
+    # Rucio decides that a multihop transfer is needed DISK1 --> DISK2 --> TAPE1 in order to put the file on tape. For some reason, the file already exists on DISK2.
+    # Rucio doesn't know about this file on DISK2. It could be either a correct or corrupted file. This can be due to a previous issue on Rucio side, FTS side, network side, etc (many possible reasons).
+    # Normally, Rucio allows overwrite towards any disk destination, but denies overwrite towards a tape destination. However, in this case, because the destination of the multihop is a tape, DISK2 cannot be overwritten.
+    # Proposed solution
+    # Provide an --overwrite-hop submission option, which instructs FTS to overwrite all transfers except for the destination within a multihop submission.
+
+    # direct transfers, is not multihop
+    if len(transfer_path) == 1:
+        overwrite_hop = False
+
+    else:
+        # Set `overwrite_hop` to `True` only if all hops allow it
+        overwrite_hop = all(transfer_hop.rws.attributes.get('overwrite', True) for transfer_hop in transfer_path[:-1])
+        job_params['overwrite'] = overwrite_hop and job_params['overwrite']
+
     if not job_params['overwrite'] and overwrite_hop:
         job_params['overwrite_hop'] = overwrite_hop
 
+    logger(logging.DEBUG, 'Job parameters are : %s' % (job_params))
     return job_params
 
 
-def bulk_group_transfers(transfer_paths, policy='rule', group_bulk=200, source_strategy=None, max_time_in_queue=None,
-                         logger=logging.log, archive_timeout_override=None, bring_online=None, default_lifetime=None):
+def bulk_group_transfers(
+        transfer_paths: "Iterable[list[DirectTransfer]]",
+        policy: str = 'rule',
+        group_bulk: int = 200,
+        source_strategy: Optional[str] = None,
+        max_time_in_queue: Optional[dict] = None,
+        logger: "LoggerFunction" = logging.log,
+        archive_timeout_override: Optional[int] = None,
+        bring_online: Optional[int] = None,
+        default_lifetime: Optional[int] = None) -> list[dict[str, Any]]:
     """
-    Group transfers in bulk based on certain criterias
+    Group transfers in bulk based on certain criteria
 
     :param transfer_paths:           List of transfer paths to group. Each path is a list of single-hop transfers.
     :param policy:                   Policy to use to group.
@@ -433,6 +495,7 @@ def bulk_group_transfers(transfer_paths, policy='rule', group_bulk=200, source_s
             max_time_in_queue=max_time_in_queue,
             logger=logger
         )
+        logger(logging.DEBUG, 'bulk_group_transfers: Job parameters are: %s' % (job_params))
         if job_params['job_metadata'].get('multi_sources') or job_params['job_metadata'].get('multihop'):
             # for multi-hop and multi-source transfers, no bulk submission.
             fts_jobs.append({'transfers': transfer_path[0:group_bulk], 'job_params': job_params})
@@ -478,6 +541,7 @@ def bulk_group_transfers(transfer_paths, policy='rule', group_bulk=200, source_s
     # split transfer groups to have at most group_bulk elements in each one
     for group in grouped_transfers.values():
         job_params = group['job_params']
+        logger(logging.DEBUG, 'bulk_group_transfers: grouped_transfers.values(): Job parameters are: %s' % (job_params))
         for transfer_paths in chunks(group['transfers'], group_bulk):
             fts_jobs.append({'transfers': transfer_paths, 'job_params': job_params})
 
@@ -498,7 +562,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
         'attributes',
     ]
 
-    def __init__(self, external_host, request_id, request=None):
+    def __init__(self, external_host: str, request_id: str, request: Optional[dict] = None):
         super().__init__(request_id, request=request)
         self.external_host = external_host
 
@@ -512,7 +576,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
         self._reason = None
         self._src_rse = None
         self._fts_address = self.external_host
-        # Supported db fields bellow:
+        # Supported db fields below:
         self.state = None
         self.external_id = None
         self.started_at = None
@@ -527,10 +591,10 @@ class Fts3TransferStatusReport(TransferStatusReport):
         return f'Transfer {self._transfer_id} of {self._file_metadata["scope"]}:{self._file_metadata["name"]} ' \
                f'{self._file_metadata["src_rse"]} --({self._file_metadata["request_id"]})-> {self._file_metadata["dst_rse"]}'
 
-    def initialize(self, session, logger=logging.log):
+    def initialize(self, session: "Session", logger: "LoggerFunction" = logging.log) -> None:
         raise NotImplementedError(f"{self.__class__.__name__} is abstract and shouldn't be used directly")
 
-    def get_monitor_msg_fields(self, session, logger=logging.log):
+    def get_monitor_msg_fields(self, session: "Session", logger: "LoggerFunction" = logging.log) -> dict[str, Any]:
         self.ensure_initialized(session, logger)
         fields = {
             'transfer_link': self._transfer_link(),
@@ -546,10 +610,10 @@ class Fts3TransferStatusReport(TransferStatusReport):
         }
         return fields
 
-    def _transfer_link(self):
+    def _transfer_link(self) -> str:
         return '%s/fts3/ftsmon/#/job/%s' % (self._fts_address.replace('8446', '8449'), self._transfer_id)
 
-    def _find_attribute_updates(self, request, new_state, reason, overwrite_corrupted_files):
+    def _find_attribute_updates(self, request: dict, new_state: RequestState, reason: str, overwrite_corrupted_files: Optional[bool] = None) -> Optional[dict[str, Any]]:
         attributes = None
         if new_state == RequestState.FAILED and 'Destination file exists and overwrite is not enabled' in (reason or ''):
             dst_file = self._file_metadata.get('dst_file', {})
@@ -559,7 +623,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
                     attributes['overwrite'] = True
         return attributes
 
-    def _find_used_source_rse(self, session, logger):
+    def _find_used_source_rse(self, session: "Session", logger: "LoggerFunction") -> tuple[Optional[str], Optional[str]]:
         """
         For multi-source transfers, FTS has a choice between multiple sources.
         Find which of the possible sources FTS actually used for the transfer.
@@ -577,7 +641,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
         return meta_rse_name, meta_rse_id
 
     @staticmethod
-    def _dst_file_set_and_file_corrupted(request, dst_file):
+    def _dst_file_set_and_file_corrupted(request: dict, dst_file: dict) -> bool:
         """
         Returns True if the `dst_file` dict returned by fts was filled and its content allows to
         affirm that the file is corrupted.
@@ -590,7 +654,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
         return False
 
     @staticmethod
-    def _dst_file_set_and_file_correct(request, dst_file):
+    def _dst_file_set_and_file_correct(request: dict, dst_file: dict) -> bool:
         """
         Returns True if the `dst_file` dict returned by fts was filled and its content allows to
         affirm that the file is correct.
@@ -635,19 +699,21 @@ class FTS3CompletionMessageTransferStatusReport(Fts3TransferStatusReport):
     """
     Parses FTS Completion messages received via the message queue
     """
-    def __init__(self, external_host, request_id, fts_message):
+    def __init__(self, external_host: str, request_id: str, fts_message: dict[str, Any]):
         super().__init__(external_host=external_host, request_id=request_id)
 
         self.fts_message = fts_message
 
-        self._transfer_id = fts_message.get('tr_id').split("__")[-1]
+        transfer_id = fts_message.get('tr_id')
+        if transfer_id is not None:
+            self._transfer_id = transfer_id.split("__")[-1]
 
         self._file_metadata = fts_message['file_metadata']
-        self._multi_sources = str(fts_message.get('job_metadata', {}).get('multi_sources', '')).lower() == str('true')
+        self._multi_sources = str(fts_message.get('job_metadata', {}).get('multi_sources', '')).lower() == 'true'
         self._src_url = fts_message.get('src_url', None)
         self._dst_url = fts_message.get('dst_url', None)
 
-    def initialize(self, session, logger=logging.log):
+    def initialize(self, session: "Session", logger: "LoggerFunction" = logging.log) -> None:
 
         fts_message = self.fts_message
         request_id = self.request_id
@@ -659,10 +725,11 @@ class FTS3CompletionMessageTransferStatusReport(Fts3TransferStatusReport):
             new_state = RequestState.DONE
         elif str(fts_message['t_final_transfer_state']) == FTS_COMPLETE_STATE.ERROR:
             request = self.request(session)
-            if self._is_recoverable_fts_overwrite_error(request, reason, self._file_metadata):  # pylint:disable=no-member
-                new_state = RequestState.DONE
-            else:
-                new_state = RequestState.FAILED
+            if request is not None:
+                if self._is_recoverable_fts_overwrite_error(request, reason, self._file_metadata):  # pylint:disable=no-member
+                    new_state = RequestState.DONE
+                else:
+                    new_state = RequestState.FAILED
 
         transfer_id = self._transfer_id
         if new_state:
@@ -705,7 +772,7 @@ class FTS3ApiTransferStatusReport(Fts3TransferStatusReport):
     """
     Parses FTS api response
     """
-    def __init__(self, external_host, request_id, job_response, file_response, request=None):
+    def __init__(self, external_host: str, request_id: str, job_response: dict[str, Any], file_response: dict[str, Any], request: Optional[dict[str, Any]] = None):
         super().__init__(external_host=external_host, request_id=request_id, request=request)
 
         self.job_response = job_response
@@ -714,12 +781,12 @@ class FTS3ApiTransferStatusReport(Fts3TransferStatusReport):
         self._transfer_id = job_response.get('job_id')
 
         self._file_metadata = file_response['file_metadata']
-        self._multi_sources = str(job_response['job_metadata'].get('multi_sources', '')).lower() == str('true')
+        self._multi_sources = str(job_response['job_metadata'].get('multi_sources', '')).lower() == 'true'
         self._src_url = file_response.get('source_surl', None)
         self._dst_url = file_response.get('dest_surl', None)
         self.logger = logging.log
 
-    def initialize(self, session, logger=logging.log):
+    def initialize(self, session: "Session", logger=logging.log) -> None:
 
         self.logger = logger
         job_response = self.job_response
@@ -739,10 +806,12 @@ class FTS3ApiTransferStatusReport(Fts3TransferStatusReport):
                 new_state = RequestState.DONE
             elif file_state == FTS_STATE.FAILED and job_state_is_final or \
                     file_state == FTS_STATE.FAILED and not self._multi_sources:  # for multi-source transfers we must wait for the job to be in a final state
-                if self._is_recoverable_fts_overwrite_error(self.request(session), reason, self._file_metadata):
-                    new_state = RequestState.DONE
-                else:
-                    new_state = RequestState.FAILED
+                request = self.request(session)
+                if request is not None:
+                    if self._is_recoverable_fts_overwrite_error(request, reason, self._file_metadata):
+                        new_state = RequestState.DONE
+                    else:
+                        new_state = RequestState.FAILED
             elif job_state_is_final and file_state == FTS_STATE.CANCELED:
                 new_state = RequestState.FAILED
             elif job_state_is_final and file_state == FTS_STATE.NOT_USED:
@@ -795,16 +864,27 @@ class FTS3Transfertool(Transfertool):
     """
 
     external_name = 'fts3'
-    required_rse_attrs = ('fts', )
-
-    def __init__(self, external_host, oidc_account=None, vo=None, group_bulk=1, group_policy='rule', source_strategy=None,
-                 max_time_in_queue=None, bring_online=43200, default_lifetime=172800, archive_timeout_override=None,
-                 logger=logging.log):
+    required_rse_attrs = (RseAttr.FTS, )
+    supported_schemes = Transfertool.supported_schemes.union(('mock', ))
+
+    def __init__(self,
+                 external_host: str,
+                 oidc_account: Optional[str] = None,
+                 oidc_support: bool = False,
+                 vo: Optional[str] = None,
+                 group_bulk: int = 1,
+                 group_policy: str = 'rule',
+                 source_strategy: Optional[str] = None,
+                 max_time_in_queue: Optional[dict[str, Any]] = None,
+                 bring_online: Optional[int] = 43200,
+                 default_lifetime: Optional[int] = 172800,
+                 archive_timeout_override: Optional[int] = None,
+                 logger: "LoggerFunction" = logging.log
+                 ):
         """
         Initializes the transfertool
 
         :param external_host:   The external host where the transfertool API is running
-        :param oidc_account:    optional oidc account to use for submission
         """
         super().__init__(external_host, logger)
 
@@ -816,20 +896,23 @@ class FTS3Transfertool(Transfertool):
         self.default_lifetime = default_lifetime
         self.archive_timeout_override = archive_timeout_override
 
-        # token for OAuth 2.0 OIDC authorization scheme (working only with dCache + davs/https protocols as of Sep 2019)
+        try:
+            tape_plugins = config_get_list("transfers", "fts3tape_metadata_plugins", raise_exception=True)
+            self.tape_metadata_plugins = [FTS3TapeMetadataPlugin(plugin.strip(" ")) for plugin in tape_plugins]
+        except (NoOptionError, NoSectionError, ValueError) as e:
+            self.logger(logging.DEBUG, f"Failed to set up any fts3 archive-metadata plugins: {e}")
+            self.tape_metadata_plugins = []
+
         self.token = None
-        if oidc_account:
-            getadmintoken = False
-            if ALLOW_USER_OIDC_TOKENS is False:
-                getadmintoken = True
-            self.logger(logging.DEBUG, 'Attempting to get a token for account %s. Admin token option set to %s' % (oidc_account, getadmintoken))
-            # find the appropriate OIDC token and exchange it (for user accounts) if necessary
-            token_dict = get_token_for_account_operation(oidc_account, req_audience=REQUEST_OIDC_AUDIENCE, req_scope=REQUEST_OIDC_SCOPE, admin=getadmintoken)
-            if token_dict is not None:
-                self.logger(logging.DEBUG, 'Access token has been granted.')
-                if 'token' in token_dict:
-                    self.logger(logging.DEBUG, 'Access token used as transfer token.')
-                    self.token = token_dict['token']
+        if oidc_support:
+            fts_hostname = urlparse(external_host).hostname
+            if fts_hostname is not None:
+                token = request_token(audience=fts_hostname, scope='fts')
+                if token is not None:
+                    self.logger(logging.INFO, 'Using a token to authenticate with FTS instance %s', fts_hostname)
+                    self.token = token
+                else:
+                    self.logger(logging.WARNING, 'Failed to procure a token to authenticate with FTS instance %s', fts_hostname)
 
         self.deterministic_id = config_get_bool('conveyor', 'use_deterministic_id', False, False)
         self.headers = {'Content-Type': 'application/json'}
@@ -849,29 +932,29 @@ class FTS3Transfertool(Transfertool):
         self.scitags_exp_id, self.scitags_activity_ids = _scitags_ids(logger=logger)
 
     @classmethod
-    def _pick_fts_servers(cls, source_rse: "RseData", dest_rse: "RseData"):
+    def _pick_fts_servers(cls, source_rse: "RseData", dest_rse: "RseData") -> Optional[list[str]]:
         """
         Pick fts servers to use for submission between the two given rse
         """
-        source_servers = source_rse.attributes.get('fts', None)
-        dest_servers = dest_rse.attributes.get('fts', None)
+        source_servers = source_rse.attributes.get(RseAttr.FTS, None)
+        dest_servers = dest_rse.attributes.get(RseAttr.FTS, None)
         if source_servers is None or dest_servers is None:
             return None
 
         servers_to_use = dest_servers
-        if source_rse.attributes.get('sign_url', None) == 'gcs':
+        if source_rse.attributes.get(RseAttr.SIGN_URL, None) == 'gcs':
             servers_to_use = source_servers
 
         return servers_to_use.split(',')
 
     @classmethod
-    def can_perform_transfer(cls, source_rse: "RseData", dest_rse: "RseData"):
+    def can_perform_transfer(cls, source_rse: "RseData", dest_rse: "RseData") -> bool:
         if cls._pick_fts_servers(source_rse, dest_rse):
             return True
         return False
 
     @classmethod
-    def submission_builder_for_path(cls, transfer_path, logger=logging.log):
+    def submission_builder_for_path(cls, transfer_path: "list[DirectTransfer]", logger: "LoggerFunction" = logging.log):
         vo = None
         if config_get_bool('common', 'multi_vo', False, None):
             vo = transfer_path[-1].rws.scope.vo
@@ -890,15 +973,15 @@ class FTS3Transfertool(Transfertool):
             logger(logging.INFO, 'FTS3Transfertool can only submit {} hops from {}'.format(len(sub_path), [str(hop) for hop in transfer_path]))
 
         if sub_path:
-            oidc_account = None
-            if all(oidc_supported(t) for t in sub_path):
+            oidc_support = False
+            if all(_use_tokens(t) for t in sub_path):
                 logger(logging.DEBUG, 'OAuth2/OIDC available for transfer {}'.format([str(hop) for hop in sub_path]))
-                oidc_account = transfer_path[-1].rws.account
-            return sub_path, TransferToolBuilder(cls, external_host=fts_hosts[0], oidc_account=oidc_account, vo=vo)
+                oidc_support = True
+            return sub_path, TransferToolBuilder(cls, external_host=fts_hosts[0], oidc_support=oidc_support, vo=vo)
         else:
             return [], None
 
-    def group_into_submit_jobs(self, transfer_paths):
+    def group_into_submit_jobs(self, transfer_paths: "list[list[DirectTransfer]]") -> list[dict[str, Any]]:
         jobs = bulk_group_transfers(
             transfer_paths,
             policy=self.group_policy,
@@ -912,11 +995,11 @@ class FTS3Transfertool(Transfertool):
         )
         return jobs
 
-    def _file_from_transfer(self, transfer, job_params):
+    def _file_from_transfer(self, transfer: "DirectTransfer", job_params: dict[str, str]) -> dict[str, Any]:
         rws = transfer.rws
         checksum_to_use = _pick_fts_checksum(transfer, path_strategy=job_params['verify_checksum'])
         t_file = {
-            'sources': [s[1] for s in transfer.legacy_sources],
+            'sources': [transfer.source_url(s) for s in transfer.sources],
             'destinations': [transfer.dest_url],
             'metadata': {
                 'request_id': rws.request_id,
@@ -939,13 +1022,32 @@ class FTS3Transfertool(Transfertool):
             'selection_strategy': self.source_strategy if self.source_strategy else _configured_source_strategy(transfer.rws.activity, logger=self.logger),
             'activity': rws.activity
         }
+
+        if self.token:
+            t_file['source_tokens'] = []
+            for source in transfer.sources:
+                src_audience = determine_audience_for_rse(rse_id=source.rse.id)
+                src_scope = determine_scope_for_rse(rse_id=source.rse.id, scopes=['storage.read'], extra_scopes=['offline_access'])
+                t_file['source_tokens'].append(request_token(src_audience, src_scope))
+
+            dst_audience = determine_audience_for_rse(transfer.dst.rse.id)
+            # FIXME: At the time of writing, StoRM requires `storage.read` in
+            # order to perform a stat operation.
+            dst_scope = determine_scope_for_rse(transfer.dst.rse.id, scopes=['storage.modify', 'storage.read'], extra_scopes=['offline_access'])
+            t_file['destination_tokens'] = [request_token(dst_audience, dst_scope)]
+
         if isinstance(self.scitags_exp_id, int):
             activity_id = self.scitags_activity_ids.get(rws.activity)
             if isinstance(activity_id, int):
                 t_file['scitag'] = self.scitags_exp_id << 6 | activity_id
+
+        if t_file['metadata']['dst_type'] == 'TAPE':
+            for plugin in self.tape_metadata_plugins:
+                t_file = deep_merge_dict(source=plugin.hints(t_file['metadata']), destination=t_file)
+
         return t_file
 
-    def submit(self, transfers, job_params, timeout=None):
+    def submit(self, transfers: "Sequence[DirectTransfer]", job_params: dict[str, str], timeout: Optional[int] = None) -> str:
         """
         Submit transfers to FTS3 via JSON.
 
@@ -1031,7 +1133,7 @@ class FTS3Transfertool(Transfertool):
         METRICS.timer('submit_transfers_fts3').observe(stopwatch.elapsed / (len(transfers) or 1))
         return transfer_id
 
-    def cancel(self, transfer_ids, timeout=None):
+    def cancel(self, transfer_ids: "Sequence[str]", timeout: Optional[int] = None) -> dict[str, Any]:
         """
         Cancel transfers that have been submitted to FTS3.
 
@@ -1059,7 +1161,7 @@ class FTS3Transfertool(Transfertool):
         CANCEL_COUNTER.labels(state='failure', host=self.__extract_host(self.external_host)).inc()
         raise Exception('Could not cancel transfer: %s', job.content)
 
-    def update_priority(self, transfer_id, priority, timeout=None):
+    def update_priority(self, transfer_id: str, priority: int, timeout: Optional[int] = None) -> dict[str, Any]:
         """
         Update the priority of a transfer that has been submitted to FTS via JSON.
 
@@ -1087,7 +1189,7 @@ class FTS3Transfertool(Transfertool):
         UPDATE_PRIORITY_COUNTER.labels(state='failure', host=self.__extract_host(self.external_host)).inc()
         raise Exception('Could not update priority of transfer: %s', job.content)
 
-    def query(self, transfer_ids, details=False, timeout=None):
+    def query(self, transfer_ids: "Sequence[str]", details: bool = False, timeout: Optional[int] = None) -> Union[Optional[dict[str, Any]], list[dict[str, Any]]]:
         """
         Query the status of a transfer in FTS3 via JSON.
 
@@ -1120,7 +1222,7 @@ class FTS3Transfertool(Transfertool):
 
     # Public methods, not part of the common interface specification (FTS3 specific)
 
-    def whoami(self):
+    def whoami(self) -> dict[str, Any]:
         """
         Returns credential information from the FTS3 server.
 
@@ -1141,7 +1243,7 @@ class FTS3Transfertool(Transfertool):
         WHOAMI_COUNTER.labels(state='failure', host=self.__extract_host(self.external_host)).inc()
         raise Exception('Could not retrieve credentials: %s', get_result.content)
 
-    def version(self):
+    def version(self) -> dict[str, Any]:
         """
         Returns FTS3 server information.
 
@@ -1162,7 +1264,7 @@ class FTS3Transfertool(Transfertool):
         VERSION_COUNTER.labels(state='failure', host=self.__extract_host(self.external_host)).inc()
         raise Exception('Could not retrieve version: %s', get_result.content)
 
-    def bulk_query(self, requests_by_eid, timeout=None):
+    def bulk_query(self, requests_by_eid: dict[str, dict[str, dict[str, Any]]], timeout: Optional[int] = None) -> dict[str, Any]:
         """
         Query the status of a bulk of transfers in FTS3 via JSON.
 
@@ -1201,11 +1303,11 @@ class FTS3Transfertool(Transfertool):
 
         return responses
 
-    def list_se_status(self):
+    def list_se_status(self) -> dict[str, Any]:
         """
         Get the list of banned Storage Elements.
 
-        :returns: Detailed dictionnary of banned Storage Elements.
+        :returns: Detailed dictionary of banned Storage Elements.
         """
 
         try:
@@ -1220,7 +1322,7 @@ class FTS3Transfertool(Transfertool):
             return result.json()
         raise Exception('Could not retrieve transfer information: %s', result.content)
 
-    def get_se_config(self, storage_element):
+    def get_se_config(self, storage_element: str) -> dict[str, Any]:
         """
         Get the Json response for the configuration of a storage element.
         :returns: a Json result for the configuration of a storage element.
@@ -1241,7 +1343,15 @@ class FTS3Transfertool(Transfertool):
             return config_se
         raise Exception('Could not get the configuration of %s , status code returned : %s', (storage_element, result.status_code if result else None))
 
-    def set_se_config(self, storage_element, inbound_max_active=None, outbound_max_active=None, inbound_max_throughput=None, outbound_max_throughput=None, staging=None):
+    def set_se_config(
+            self,
+            storage_element: str,
+            inbound_max_active: Optional[int] = None,
+            outbound_max_active: Optional[int] = None,
+            inbound_max_throughput: Optional[float] = None,
+            outbound_max_throughput: Optional[float] = None,
+            staging: Optional[int] = None
+    ) -> dict[str, Any]:
         """
         Set the configuration for a storage element. Used for alleviating transfer failures due to timeout.
 
@@ -1301,7 +1411,7 @@ class FTS3Transfertool(Transfertool):
             return configSe
         raise Exception('Could not set the configuration of %s , status code returned : %s', (storage_element, result.status_code if result else None))
 
-    def set_se_status(self, storage_element, message, ban=True, timeout=None):
+    def set_se_status(self, storage_element: str, message: str, ban: bool = True, timeout: Optional[int] = None) -> int:
         """
         Ban a Storage Element. Used when a site is in downtime.
         One can use a timeout in seconds. In that case the jobs will wait before being cancel.
@@ -1315,7 +1425,7 @@ class FTS3Transfertool(Transfertool):
         :returns: 0 in case of success, otherwise raise Exception
         """
 
-        params_dict = {'storage': storage_element, 'message': message}
+        params_dict: dict[str, Any] = {'storage': storage_element, 'message': message}
         status = 'CANCEL'
         if timeout:
             params_dict['timeout'] = timeout
@@ -1355,11 +1465,13 @@ class FTS3Transfertool(Transfertool):
     # Private methods unique to the FTS3 Transfertool
 
     @staticmethod
-    def __extract_host(external_host):
+    def __extract_host(external_host: str) -> Optional[str]:
         # graphite does not like the dots in the FQDN
-        return urlparse(external_host).hostname.replace('.', '_')
+        parsed_url = urlparse(external_host)
+        if parsed_url.hostname:
+            return parsed_url.hostname.replace('.', '_')
 
-    def __get_transfer_baseid_voname(self):
+    def __get_transfer_baseid_voname(self) -> tuple[Optional[str], Optional[str]]:
         """
         Get transfer VO name from the external host.
 
@@ -1402,7 +1514,7 @@ class FTS3Transfertool(Transfertool):
             result = (None, None)
         return result
 
-    def __get_deterministic_id(self, sid):
+    def __get_deterministic_id(self, sid: str) -> Optional[str]:
         """
         Get deterministic FTS job id.
 
@@ -1417,7 +1529,7 @@ class FTS3Transfertool(Transfertool):
         jobid = uuid.uuid5(atlas, sid)
         return str(jobid)
 
-    def __bulk_query_responses(self, jobs_response, requests_by_eid):
+    def __bulk_query_responses(self, jobs_response: list[dict[str, Any]], requests_by_eid: dict[str, dict[str, dict[str, Any]]]) -> dict[str, Any]:
         if not isinstance(jobs_response, list):
             jobs_response = [jobs_response]
 
@@ -1431,7 +1543,7 @@ class FTS3Transfertool(Transfertool):
                                                                        FTS_STATE.FINISHEDDIRTY,
                                                                        FTS_STATE.CANCELED,
                                                                        FTS_STATE.FINISHED]:
-                    # multipe source replicas jobs is still running. should wait
+                    # multiple source replicas jobs is still running. should wait
                     responses[transfer_id] = {}
                     continue
 
@@ -1461,7 +1573,7 @@ class FTS3Transfertool(Transfertool):
                                                                                                                                    job_response['http_message'] if 'http_message' in job_response else None))
         return responses
 
-    def __query_details(self, transfer_id):
+    def __query_details(self, transfer_id: str) -> Optional[dict[str, Any]]:
         """
         Query the detailed status of a transfer in FTS3 via JSON.