rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/daemons/conveyor/poller.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,9 +23,9 @@ import logging
 import re
 import threading
 import time
+from collections.abc import Mapping, Sequence
 from itertools import groupby
-from types import FrameType
-from typing import TYPE_CHECKING, Mapping, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Optional
 
 from requests.exceptions import RequestException
 from sqlalchemy.exc import DatabaseError
@@ -36,18 +35,20 @@ from rucio.common.config import config_get, config_get_bool
 from rucio.common.exception import DatabaseException, TransferToolTimeout, TransferToolWrongAnswer
 from rucio.common.logging import setup_logging
 from rucio.common.stopwatch import Stopwatch
-from rucio.common.types import InternalAccount
+from rucio.common.types import InternalAccount, LoggerFunction
 from rucio.common.utils import dict_chunks
-from rucio.core import transfer as transfer_core, request as request_core
+from rucio.core import request as request_core
+from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MetricManager
-from rucio.core.topology import Topology, ExpiringObjectCache
-from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
-from rucio.db.sqla.constants import RequestState, RequestType
+from rucio.core.topology import ExpiringObjectCache, Topology
+from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
+from rucio.db.sqla.constants import MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED, ORACLE_DEADLOCK_DETECTED_REGEX, ORACLE_RESOURCE_BUSY_REGEX, RequestState, RequestType
 from rucio.transfertool.fts3 import FTS3Transfertool
-from rucio.transfertool.globus import GlobusTransferTool
-from rucio.transfertool.mock import MockTransfertool
+from rucio.transfertool.transfertool import Transfertool
 
 if TYPE_CHECKING:
+    from types import FrameType
+
     from rucio.daemons.common import HeartbeatHandler
 
 GRACEFUL_STOP = threading.Event()
@@ -59,16 +60,16 @@ FILTER_TRANSFERTOOL = config_get('conveyor', 'filter_transfertool', False, None)
 
 
 def _fetch_requests(
-        db_bulk,
-        older_than,
-        activity_shares,
-        transfertool,
-        filter_transfertool,
-        cached_topology,
-        activity,
+        db_bulk: int,
+        older_than: int,
+        activity_shares: Optional[Mapping[str, float]],
+        transfertool: Optional[str],
+        filter_transfertool: Optional[str],
+        cached_topology: Optional[ExpiringObjectCache],
+        activity: str,
         set_last_processed_by: bool,
-        heartbeat_handler
-):
+        heartbeat_handler: "HeartbeatHandler"
+) -> tuple[bool, list[dict[str, Any]]]:
     worker_number, total_workers, logger = heartbeat_handler.live()
 
     logger(logging.DEBUG, 'Start to poll transfers older than %i seconds for activity %s using transfer tool: %s' % (older_than, activity, filter_transfertool))
@@ -94,7 +95,7 @@ def _fetch_requests(
         # only keep transfers which don't have any transfertool set, or have one equal to TRANSFER_TOOL
         transfs_tmp = [t for t in transfs if not t['transfertool'] or t['transfertool'] == transfertool]
         if len(transfs_tmp) != len(transfs):
-            logger(logging.INFO, 'Skipping %i transfers because of missmatched transfertool', len(transfs) - len(transfs_tmp))
+            logger(logging.INFO, 'Skipping %i transfers because of mismatched transfertool', len(transfs) - len(transfs_tmp))
         transfs = transfs_tmp
 
     if transfs:
@@ -109,15 +110,16 @@ def _fetch_requests(
 
 
 def _handle_requests(
-        transfs,
-        fts_bulk,
-        multi_vo,
-        timeout,
-        transfertool,
+        transfs: list[dict[str, Any]],
+        fts_bulk: int,
+        multi_vo: Optional[bool],
+        timeout: Optional[int],
+        transfertool: str,
+        transfer_stats_manager: request_core.TransferStatsManager,
         oidc_account: Optional[str],
         *,
-        logger=logging.log,
-):
+        logger: LoggerFunction = logging.log,
+) -> None:
     transfs.sort(key=lambda t: (t['external_host'] or '',
                                 t['scope'].vo if multi_vo else '',
                                 t['external_id'] or '',
@@ -130,19 +132,29 @@ def _handle_requests(
 
         for chunk in dict_chunks(transfers_by_eid, fts_bulk):
             try:
-                if transfertool == 'mock':
-                    transfertool_obj = MockTransfertool(external_host=MockTransfertool.external_name)
-                elif transfertool == 'globus':
-                    transfertool_obj = GlobusTransferTool(external_host=GlobusTransferTool.external_name)
-                else:
+                transfertool_cls = transfer_core.TRANSFERTOOL_CLASSES_BY_NAME.get(transfertool, FTS3Transfertool)
+
+                transfertool_kwargs = {}
+                if transfertool_cls.external_name == FTS3Transfertool.external_name:
                     account = None
                     if oidc_account:
                         if vo:
                             account = InternalAccount(oidc_account, vo=vo)
                         else:
                             account = InternalAccount(oidc_account)
-                    transfertool_obj = FTS3Transfertool(external_host=external_host, vo=vo, oidc_account=account)
-                poll_transfers(transfertool_obj=transfertool_obj, transfers_by_eid=chunk, timeout=timeout, logger=logger)
+                    transfertool_kwargs.update({
+                        'vo': vo,
+                        'oidc_account': account,
+                    })
+
+                transfertool_obj = transfertool_cls(external_host=external_host, **transfertool_kwargs)
+                poll_transfers(
+                    transfertool_obj=transfertool_obj,
+                    transfers_by_eid=chunk,
+                    transfer_stats_manager=transfer_stats_manager,
+                    timeout=timeout,
+                    logger=logger,
+                )
             except Exception:
                 logger(logging.ERROR, 'Exception', exc_info=True)
 
@@ -158,9 +170,9 @@ def poller(
         partition_wait_time: int = 10,
         transfertool: Optional[str] = TRANSFER_TOOL,
         filter_transfertool: Optional[str] = FILTER_TRANSFERTOOL,
-        cached_topology=None,
+        cached_topology: Optional[ExpiringObjectCache] = None,
         total_threads: int = 1,
-):
+) -> None:
     """
     Main loop to check the status of a transfer primitive with a transfertool.
     """
@@ -182,6 +194,8 @@ def poller(
     if filter_transfertool:
         executable += ' --filter-transfertool ' + filter_transfertool
 
+    transfer_stats_manager = request_core.TransferStatsManager()
+
     @db_workqueue(
         once=once,
         graceful_stop=GRACEFUL_STOP,
@@ -190,7 +204,11 @@ def poller(
         sleep_time=sleep_time,
         activities=activities,
     )
-    def _db_producer(*, activity: str, heartbeat_handler: "HeartbeatHandler"):
+    def _db_producer(
+        *,
+        activity: str,
+        heartbeat_handler: "HeartbeatHandler"
+    ) -> tuple[bool, list[dict[str, Any]]]:
         return _fetch_requests(
             db_bulk=db_bulk,
             older_than=older_than,
@@ -203,24 +221,26 @@ def poller(
             heartbeat_handler=heartbeat_handler,
         )
 
-    def _consumer(transfs):
+    def _consumer(transfs: list[dict[str, Any]]) -> None:
         return _handle_requests(
             transfs=transfs,
             fts_bulk=fts_bulk,
             multi_vo=multi_vo,
-            timeout=timeout,
+            timeout=timeout,  # type: ignore (unclear if timeout is meant to be int or float)
             oidc_account=oidc_account,
-            transfertool=transfertool,
+            transfertool=transfertool,  # type: ignore (transfertool is not None)
+            transfer_stats_manager=transfer_stats_manager,
         )
 
-    ProducerConsumerDaemon(
-        producers=[_db_producer],
-        consumers=[_consumer for _ in range(total_threads)],
-        graceful_stop=GRACEFUL_STOP,
-    ).run()
+    with transfer_stats_manager:
+        ProducerConsumerDaemon(
+            producers=[_db_producer],
+            consumers=[_consumer for _ in range(total_threads)],
+            graceful_stop=GRACEFUL_STOP,
+        ).run()
 
 
-def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -229,17 +249,17 @@ def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> Non
 
 
 def run(
-        once=False,
-        sleep_time=60,
-        activities=None,
-        fts_bulk=100,
-        db_bulk=1000,
-        older_than=60,
+        once: bool = False,
+        sleep_time: int = 60,
+        activities: Optional[Sequence[str]] = None,
+        fts_bulk: int = 100,
+        db_bulk: int = 1000,
+        older_than: int = 60,
         activity_shares: Optional[str] = None,
-        total_threads=1
-):
+        total_threads: int = 1
+) -> None:
     """
-    Starts up the conveyer threads.
+    Starts up the conveyor threads.
     """
     setup_logging(process_name=DAEMON_NAME)
 
@@ -280,19 +300,20 @@ def run(
     )
 
 
-def poll_transfers(transfertool_obj, transfers_by_eid, timeout=None, logger=logging.log):
+def poll_transfers(
+        transfertool_obj: Transfertool,
+        transfers_by_eid: Mapping[str, Mapping[str, Any]],
+        transfer_stats_manager: request_core.TransferStatsManager,
+        timeout: "Optional[int]" = None,
+        logger: "LoggerFunction" = logging.log
+) -> None:
     """
     Poll a list of transfers from an FTS server
-
-    :param transfertool_obj: The Transfertool to use for query
-    :param transfers_by_eid: Dict of the form {external_id: list_of_transfers}
-    :param timeout:          Timeout.
-    :param logger:           Optional decorated logger that can be passed from the calling daemons or servers.
     """
 
     poll_individual_transfers = False
     try:
-        _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger)
+        _poll_transfers(transfertool_obj, transfers_by_eid, transfer_stats_manager, timeout, logger)
     except TransferToolWrongAnswer:
         poll_individual_transfers = True
 
@@ -301,12 +322,18 @@ def poll_transfers(transfertool_obj, transfers_by_eid, timeout=None, logger=logg
         for external_id, transfers in transfers_by_eid.items():
             logger(logging.DEBUG, 'Checking %s on %s' % (external_id, transfertool_obj))
             try:
-                _poll_transfers(transfertool_obj, {external_id: transfers}, timeout, logger)
+                _poll_transfers(transfertool_obj, {external_id: transfers}, transfer_stats_manager, timeout, logger)
             except Exception as err:
                 logger(logging.ERROR, 'Problem querying %s on %s . Error returned : %s' % (external_id, transfertool_obj, str(err)))
 
 
-def _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger):
+def _poll_transfers(
+        transfertool_obj: Transfertool,
+        transfers_by_eid: Mapping[str, Mapping[str, Any]],
+        transfer_stats_manager: request_core.TransferStatsManager,
+        timeout: "Optional[int]" = None,
+        logger: "LoggerFunction" = logging.log
+) -> None:
     """
     Helper function for poll_transfers which performs the actual polling and database update.
     """
@@ -355,17 +382,22 @@ def _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger):
                 METRICS.counter('query_transfer_exception').inc()
             else:
                 for request_id in request_ids.intersection(transf_resp):
-                    ret = transfer_core.update_transfer_state(transf_resp[request_id], logger=logger)
-                    # if True, really update request content; if False, only touch request
+                    ret = transfer_core.update_transfer_state(
+                        tt_status_report=transf_resp[request_id],
+                        stats_manager=transfer_stats_manager,
+                        logger=logger,
+                    )
+                    cnt += ret
                     if ret:
-                        cnt += 1
-                    METRICS.counter('update_request_state.{updated}').labels(updated=ret).inc()
+                        METRICS.counter('update_request_state.{updated}').labels(updated=True).inc(delta=ret)
+                    else:
+                        METRICS.counter('update_request_state.{updated}').labels(updated=False).inc()
 
             # should touch transfers.
             # Otherwise if one bulk transfer includes many requests and one is not terminated, the transfer will be poll again.
             transfer_core.touch_transfer(transfertool_obj.external_host, transfer_id)
         except (DatabaseException, DatabaseError) as error:
-            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+            if re.match(ORACLE_RESOURCE_BUSY_REGEX, error.args[0]) or re.match(ORACLE_DEADLOCK_DETECTED_REGEX, error.args[0]) or MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED in error.args[0]:
                 logger(logging.WARNING, "Lock detected when handling request %s - skipping" % transfer_id)
             else:
                 logger(logging.ERROR, 'Exception', exc_info=True)

rucio/daemons/conveyor/preparer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,6 @@
 import logging
 import threading
 from time import time
-from types import FrameType
 from typing import TYPE_CHECKING, Optional
 
 import rucio.db.sqla.util
@@ -25,21 +23,26 @@ from rucio.common.config import config_get_list
 from rucio.common.exception import RucioException
 from rucio.common.logging import setup_logging
 from rucio.core import transfer as transfer_core
-from rucio.core.request import set_requests_state_if_possible, list_and_mark_transfer_requests_and_source_replicas
-from rucio.core.topology import Topology, ExpiringObjectCache
-from rucio.core.transfer import prepare_transfers, list_transfer_admin_accounts, build_transfer_paths, ProtocolFactory
-from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
+from rucio.core.request import RequestWithSources, list_and_mark_transfer_requests_and_source_replicas, transition_requests_state_if_possible
+from rucio.core.topology import ExpiringObjectCache, Topology
+from rucio.core.transfer import ProtocolFactory, build_transfer_paths, list_transfer_admin_accounts, prepare_transfers
+from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
 from rucio.db.sqla.constants import RequestState, RequestType
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from types import FrameType
+
     from sqlalchemy.orm import Session
+
+    from rucio.common.types import LoggerFunction
     from rucio.daemons.common import HeartbeatHandler
 
 GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'conveyor-preparer'
 
 
-def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -48,12 +51,12 @@ def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> Non
 
 
 def run(
-        once=False,
-        threads=1,
-        sleep_time=10,
-        bulk=100,
+        once: bool = False,
+        threads: int = 1,
+        sleep_time: int = 10,
+        bulk: int = 100,
         ignore_availability: bool = False
-):
+) -> None:
     """
     Running the preparer daemon either once or by default in a loop until stop is called.
     """
@@ -75,15 +78,15 @@ def run(
 
 
 def preparer(
-        once,
+        once: bool,
         sleep_time: int = 10,
         bulk: int = 100,
         ignore_availability: bool = False,
         partition_wait_time: int = 10,
-        transfertools=None,
-        cached_topology=None,
-        total_threads=1
-):
+        transfertools: Optional[list[str]] = None,
+        cached_topology: Optional[ExpiringObjectCache] = None,
+        total_threads: int = 1
+) -> None:
     # Make an initial heartbeat so that all instanced daemons have the correct worker number on the next try
     executable = DAEMON_NAME
     if not transfertools:
@@ -95,7 +98,11 @@ def preparer(
         executable=executable,
         partition_wait_time=partition_wait_time,
         sleep_time=sleep_time)
-    def _db_producer(*, activity: str, heartbeat_handler: "HeartbeatHandler"):
+    def _db_producer(
+        *,
+        activity: str,
+        heartbeat_handler: "HeartbeatHandler"
+    ) -> tuple[bool, tuple[Topology, dict[str, RequestWithSources]]]:
         return _fetch_requests(
             bulk=bulk,
             ignore_availability=ignore_availability,
@@ -104,7 +111,7 @@ def preparer(
             set_last_processed_by=not once,
         )
 
-    def _consumer(batch):
+    def _consumer(batch: tuple[Topology, "Mapping[str, RequestWithSources]"]) -> None:
         return _handle_requests(
             batch,
             transfertools=transfertools,
@@ -121,12 +128,12 @@ def preparer(
 def _fetch_requests(
         bulk: int,
         ignore_availability: bool,
-        cached_topology,
+        cached_topology: Optional[ExpiringObjectCache],
         heartbeat_handler: "HeartbeatHandler",
         set_last_processed_by: bool,
         *,
-        session: "Optional[Session]" = None,
-):
+        session: Optional["Session"] = None,
+) -> tuple[bool, tuple[Topology, dict[str, RequestWithSources]]]:
     worker_number, total_workers, logger = heartbeat_handler.live()
     topology = cached_topology.get() if cached_topology else Topology(ignore_availability=ignore_availability)
     topology.configure_multihop(logger=logger, session=session)
@@ -149,12 +156,12 @@ def _fetch_requests(
 
 
 def _handle_requests(
-        batch,
+        batch: tuple[Topology, "Mapping[str, RequestWithSources]"],
         *,
         transfertools: Optional[list[str]] = None,
         bulk: int = 100,
-        logger=logging.log,
-):
+        logger: "LoggerFunction" = logging.log,
+) -> None:
     topology, requests_with_sources = batch
 
     if not transfertools:
@@ -185,13 +192,13 @@ def _handle_requests(
             reqs_no_source.update(reqs_no_transfertool)
             if reqs_no_source:
                 logger(logging.INFO, "Marking requests as no-sources: %s", reqs_no_source)
-                set_requests_state_if_possible(reqs_no_source, RequestState.NO_SOURCES, logger=logger)
+                transition_requests_state_if_possible(reqs_no_source, RequestState.NO_SOURCES, logger=logger)
             if reqs_only_tape_source:
                 logger(logging.INFO, "Marking requests as only-tape-sources: %s", reqs_only_tape_source)
-                set_requests_state_if_possible(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES, logger=logger)
+                transition_requests_state_if_possible(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES, logger=logger)
             if reqs_scheme_mismatch:
                 logger(logging.INFO, "Marking requests as scheme-mismatch: %s", reqs_scheme_mismatch)
-                set_requests_state_if_possible(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME, logger=logger)
+                transition_requests_state_if_possible(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME, logger=logger)
     except RucioException:
         logger(logging.ERROR, 'errored with a RucioException, retrying later', exc_info=True)
         updated_msg = 'errored'

rucio/daemons/conveyor/receiver.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,8 +22,7 @@ import socket
 import threading
 import time
 import traceback
-from types import FrameType
-from typing import Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 import stomp
 
@@ -33,12 +31,21 @@ from rucio.common import exception
 from rucio.common.config import config_get, config_get_bool, config_get_int
 from rucio.common.logging import setup_logging
 from rucio.common.policy import get_policy
+from rucio.core import request as request_core
 from rucio.core import transfer as transfer_core
 from rucio.core.monitor import MetricManager
 from rucio.daemons.common import HeartbeatHandler
 from rucio.db.sqla.session import transactional_session
 from rucio.transfertool.fts3 import FTS3CompletionMessageTransferStatusReport
 
+if TYPE_CHECKING:
+    from types import FrameType
+
+    from sqlalchemy.orm import Session
+    from stomp.utils import Frame
+
+    from rucio.common.types import LoggerFunction
+
 logging.getLogger("stomp").setLevel(logging.CRITICAL)
 
 METRICS = MetricManager(module=__name__)
@@ -46,21 +53,29 @@ GRACEFUL_STOP = threading.Event()
 DAEMON_NAME = 'conveyor-receiver'
 
 
-class Receiver(object):
+class Receiver:
 
-    def __init__(self, broker, id_, total_threads, all_vos=False):
+    def __init__(
+            self,
+            broker: str,
+            id_: str,
+            total_threads: int,
+            transfer_stats_manager: request_core.TransferStatsManager,
+            all_vos: bool = False
+    ):
         self.__all_vos = all_vos
         self.__broker = broker
         self.__id = id_
         self.__total_threads = total_threads
+        self._transfer_stats_manager = transfer_stats_manager
 
     @METRICS.count_it
-    def on_error(self, frame):
+    def on_error(self, frame: "Frame") -> None:
         logging.error('[%s] %s' % (self.__broker, frame.body))
 
     @METRICS.count_it
-    def on_message(self, frame):
-        msg = json.loads(frame.body)
+    def on_message(self, frame: "Frame") -> None:
+        msg = json.loads(frame.body)  # type: ignore
 
         if not self.__all_vos:
             if 'vo' not in msg or msg['vo'] != get_policy():
@@ -69,29 +84,47 @@ class Receiver(object):
         if 'job_metadata' in msg.keys() \
            and isinstance(msg['job_metadata'], dict) \
            and 'issuer' in msg['job_metadata'].keys() \
-           and str(msg['job_metadata']['issuer']) == str('rucio'):
+           and str(msg['job_metadata']['issuer']) == 'rucio':
 
-            if 'job_state' in msg.keys() and (str(msg['job_state']) != str('ACTIVE') or msg.get('job_multihop', False) is True):
+            if 'job_state' in msg.keys() and (str(msg['job_state']) != 'ACTIVE' or msg.get('job_multihop', False) is True):
                 METRICS.counter('message_rucio').inc()
 
                 self._perform_request_update(msg)
 
     @transactional_session
-    def _perform_request_update(self, msg, *, session=None, logger=logging.log):
+    def _perform_request_update(
+        self,
+        msg: dict[str, Any],
+        *,
+        session: Optional["Session"] = None,
+        logger: "LoggerFunction" = logging.log
+    ) -> None:
         external_host = msg.get('endpnt', None)
         request_id = msg['file_metadata'].get('request_id', None)
         try:
             tt_status_report = FTS3CompletionMessageTransferStatusReport(external_host, request_id=request_id, fts_message=msg)
-            if tt_status_report.get_db_fields_to_update(session=session, logger=logger):
+            if tt_status_report.get_db_fields_to_update(session=session, logger=logger):  # type: ignore
                 logging.info('RECEIVED %s', tt_status_report)
 
-                ret = transfer_core.update_transfer_state(tt_status_report, session=session, logger=logger)
-                METRICS.counter('update_request_state.{updated}').labels(updated=ret).inc()
+                ret = transfer_core.update_transfer_state(
+                    tt_status_report=tt_status_report,
+                    stats_manager=self._transfer_stats_manager,
+                    session=session,
+                    logger=logger,
+                )
+                if ret:
+                    METRICS.counter('update_request_state.{updated}').labels(updated=True).inc(delta=ret)
+                else:
+                    METRICS.counter('update_request_state.{updated}').labels(updated=False).inc()
         except Exception:
             logging.critical(traceback.format_exc())
 
 
-def receiver(id_, total_threads=1, all_vos=False):
+def receiver(
+        id_: str,
+        total_threads: int = 1,
+        all_vos: bool = False
+) -> None:
     """
     Main loop to consume messages from the FTS3 producer.
     """
@@ -146,8 +179,8 @@ def receiver(id_, total_threads=1, all_vos=False):
 
     logging.info('receiver started')
 
-    with HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30) as heartbeat_handler:
-
+    with (HeartbeatHandler(executable=DAEMON_NAME, renewal_interval=30) as heartbeat_handler,
+          request_core.TransferStatsManager() as transfer_stats_manager):
         while not GRACEFUL_STOP.is_set():
 
             _, _, logger = heartbeat_handler.live()
@@ -158,8 +191,15 @@ def receiver(id_, total_threads=1, all_vos=False):
                     logger(logging.INFO, 'connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
                     METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0].split('.')[0]).inc()
 
-                    conn.set_listener('rucio-messaging-fts3', Receiver(broker=conn.transport._Transport__host_and_ports[0],
-                                                                       id_=id_, total_threads=total_threads, all_vos=all_vos))
+                    conn.set_listener(
+                        'rucio-messaging-fts3',
+                        Receiver(
+                            broker=conn.transport._Transport__host_and_ports[0],
+                            id_=id_,
+                            total_threads=total_threads,
+                            transfer_stats_manager=transfer_stats_manager,
+                            all_vos=all_vos
+                        ))
                     if not use_ssl:
                         conn.connect(username, password, wait=True)
                     else:
@@ -176,7 +216,7 @@ def receiver(id_, total_threads=1, all_vos=False):
                 pass
 
 
-def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
     """
     Graceful exit.
     """
@@ -184,7 +224,10 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
     GRACEFUL_STOP.set()
 
 
-def run(once=False, total_threads=1):
+def run(
+        once: bool = False,
+        total_threads: int = 1
+) -> None:
     """
     Starts up the receiver thread
     """