rucio 32.8.6__py3-none-any.whl

rucio/daemons/conveyor/poller.py

@@ -0,0 +1,372 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import datetime
+import itertools
+import json
+import logging
+import re
+import threading
+import time
+from itertools import groupby
+from types import FrameType
+from typing import TYPE_CHECKING, Mapping, Optional, Sequence
+
+from requests.exceptions import RequestException
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common.config import config_get, config_get_bool
+from rucio.common.exception import DatabaseException, TransferToolTimeout, TransferToolWrongAnswer
+from rucio.common.logging import setup_logging
+from rucio.common.stopwatch import Stopwatch
+from rucio.common.types import InternalAccount
+from rucio.common.utils import dict_chunks
+from rucio.core import transfer as transfer_core, request as request_core
+from rucio.core.monitor import MetricManager
+from rucio.core.topology import Topology, ExpiringObjectCache
+from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
+from rucio.db.sqla.constants import RequestState, RequestType
+from rucio.transfertool.fts3 import FTS3Transfertool
+from rucio.transfertool.globus import GlobusTransferTool
+from rucio.transfertool.mock import MockTransfertool
+
+if TYPE_CHECKING:
+    from rucio.daemons.common import HeartbeatHandler
+
+GRACEFUL_STOP = threading.Event()
+METRICS = MetricManager(module=__name__)
+DAEMON_NAME = 'conveyor-poller'
+
+TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)  # NOTE: This should eventually be completely removed, as it can be fetched from the request
+FILTER_TRANSFERTOOL = config_get('conveyor', 'filter_transfertool', False, None)  # NOTE: TRANSFERTOOL to filter requests on
+
+
+def _fetch_requests(
+        db_bulk,
+        older_than,
+        activity_shares,
+        transfertool,
+        filter_transfertool,
+        cached_topology,
+        activity,
+        set_last_processed_by: bool,
+        heartbeat_handler
+):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    logger(logging.DEBUG, 'Start to poll transfers older than %i seconds for activity %s using transfer tool: %s' % (older_than, activity, filter_transfertool))
+
+    topology = cached_topology.get() if cached_topology else Topology()
+    transfs = request_core.get_and_mark_next(
+        rse_collection=topology,
+        request_type=[RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT],
+        state=[RequestState.SUBMITTED],
+        processed_by=heartbeat_handler.short_executable if set_last_processed_by else None,
+        limit=db_bulk,
+        older_than=datetime.datetime.utcnow() - datetime.timedelta(seconds=older_than) if older_than else None,
+        total_workers=total_workers,
+        worker_number=worker_number,
+        mode_all=True,
+        hash_variable='id',
+        activity=activity,
+        activity_shares=activity_shares,
+        transfertool=filter_transfertool,
+    )
+
+    if transfertool and not filter_transfertool:
+        # only keep transfers which don't have any transfertool set, or have one equal to TRANSFER_TOOL
+        transfs_tmp = [t for t in transfs if not t['transfertool'] or t['transfertool'] == transfertool]
+        if len(transfs_tmp) != len(transfs):
+            logger(logging.INFO, 'Skipping %i transfers because of missmatched transfertool', len(transfs) - len(transfs_tmp))
+        transfs = transfs_tmp
+
+    if transfs:
+        logger(logging.DEBUG, 'Polling %i transfers for activity %s' % (len(transfs), activity))
+
+    must_sleep = False
+    if len(transfs) < db_bulk / 2:
+        logger(logging.INFO, "Only %s transfers for activity %s, which is less than half of the bulk %s" % (len(transfs), activity, db_bulk))
+        must_sleep = True
+
+    return must_sleep, transfs
+
+
+def _handle_requests(
+        transfs,
+        fts_bulk,
+        multi_vo,
+        timeout,
+        transfertool,
+        oidc_account: Optional[str],
+        *,
+        logger=logging.log,
+):
+    transfs.sort(key=lambda t: (t['external_host'] or '',
+                                t['scope'].vo if multi_vo else '',
+                                t['external_id'] or '',
+                                t['request_id'] or ''))
+    for (external_host, vo), transfers_for_host in groupby(transfs, key=lambda t: (t['external_host'],
+                                                                                   t['scope'].vo if multi_vo else None)):
+        transfers_by_eid = {}
+        for external_id, xfers in groupby(transfers_for_host, key=lambda t: t['external_id']):
+            transfers_by_eid[external_id] = {t['request_id']: t for t in xfers}
+
+        for chunk in dict_chunks(transfers_by_eid, fts_bulk):
+            try:
+                if transfertool == 'mock':
+                    transfertool_obj = MockTransfertool(external_host=MockTransfertool.external_name)
+                elif transfertool == 'globus':
+                    transfertool_obj = GlobusTransferTool(external_host=GlobusTransferTool.external_name)
+                else:
+                    account = None
+                    if oidc_account:
+                        if vo:
+                            account = InternalAccount(oidc_account, vo=vo)
+                        else:
+                            account = InternalAccount(oidc_account)
+                    transfertool_obj = FTS3Transfertool(external_host=external_host, vo=vo, oidc_account=account)
+                poll_transfers(transfertool_obj=transfertool_obj, transfers_by_eid=chunk, timeout=timeout, logger=logger)
+            except Exception:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+
+
+def poller(
+        once: bool = False,
+        activities: Optional[Sequence[str]] = None,
+        sleep_time: int = 60,
+        fts_bulk: int = 100,
+        db_bulk: int = 1000,
+        older_than: int = 60,
+        activity_shares: Optional[Mapping[str, float]] = None,
+        partition_wait_time: int = 10,
+        transfertool: Optional[str] = TRANSFER_TOOL,
+        filter_transfertool: Optional[str] = FILTER_TRANSFERTOOL,
+        cached_topology=None,
+        total_threads: int = 1,
+):
+    """
+    Main loop to check the status of a transfer primitive with a transfertool.
+    """
+
+    timeout = config_get('conveyor', 'poll_timeout', default=None, raise_exception=False)
+    if timeout:
+        timeout = float(timeout)
+
+    multi_vo = config_get_bool('common', 'multi_vo', False, None)
+    oidc_account = config_get('conveyor', 'poller_oidc_account', False, None)
+
+    executable = DAEMON_NAME
+
+    if activities:
+        activities = sorted(activities)
+        executable += '--activities ' + str(activities)
+    if activity_shares:
+        executable += '--activity_shares' + str(sorted(activity_shares))
+    if filter_transfertool:
+        executable += ' --filter-transfertool ' + filter_transfertool
+
+    @db_workqueue(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=executable,
+        partition_wait_time=partition_wait_time,
+        sleep_time=sleep_time,
+        activities=activities,
+    )
+    def _db_producer(*, activity: str, heartbeat_handler: "HeartbeatHandler"):
+        return _fetch_requests(
+            db_bulk=db_bulk,
+            older_than=older_than,
+            activity_shares=activity_shares,
+            transfertool=transfertool,
+            filter_transfertool=filter_transfertool,
+            cached_topology=cached_topology,
+            activity=activity,
+            set_last_processed_by=not once,
+            heartbeat_handler=heartbeat_handler,
+        )
+
+    def _consumer(transfs):
+        return _handle_requests(
+            transfs=transfs,
+            fts_bulk=fts_bulk,
+            multi_vo=multi_vo,
+            timeout=timeout,
+            oidc_account=oidc_account,
+            transfertool=transfertool,
+        )
+
+    ProducerConsumerDaemon(
+        producers=[_db_producer],
+        consumers=[_consumer for _ in range(total_threads)],
+        graceful_stop=GRACEFUL_STOP,
+    ).run()
+
+
+def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(
+        once=False,
+        sleep_time=60,
+        activities=None,
+        fts_bulk=100,
+        db_bulk=1000,
+        older_than=60,
+        activity_shares: Optional[str] = None,
+        total_threads=1
+):
+    """
+    Starts up the conveyer threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise DatabaseException('Database was not updated, daemon won\'t start')
+
+    parsed_activity_shares = None
+    if activity_shares:
+
+        try:
+            parsed_activity_shares = {str(activity): float(share) for activity, share in json.loads(activity_shares).items()}
+        except Exception:
+            logging.critical('activity share is not a valid JSON dictionary')
+            return
+
+        try:
+            if round(sum(parsed_activity_shares.values()), 2) != 1:
+                logging.critical('activity shares do not sum up to 1, got %s - aborting' % round(sum(parsed_activity_shares.values()), 2))
+                return
+        except Exception:
+            logging.critical('activity shares are not numbers? - aborting')
+            return
+
+        parsed_activity_shares.update((share, int(percentage * db_bulk)) for share, percentage in parsed_activity_shares.items())
+        logging.info('activity shares enabled: %s' % parsed_activity_shares)
+
+    cached_topology = ExpiringObjectCache(ttl=300, new_obj_fnc=lambda: Topology())
+    poller(
+        once=once,
+        fts_bulk=fts_bulk,
+        db_bulk=db_bulk,
+        older_than=older_than,
+        sleep_time=sleep_time,
+        activities=activities,
+        activity_shares=parsed_activity_shares,
+        cached_topology=cached_topology,
+        total_threads=total_threads,
+    )
+
+
+def poll_transfers(transfertool_obj, transfers_by_eid, timeout=None, logger=logging.log):
+    """
+    Poll a list of transfers from an FTS server
+
+    :param transfertool_obj: The Transfertool to use for query
+    :param transfers_by_eid: Dict of the form {external_id: list_of_transfers}
+    :param timeout:          Timeout.
+    :param logger:           Optional decorated logger that can be passed from the calling daemons or servers.
+    """
+
+    poll_individual_transfers = False
+    try:
+        _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger)
+    except TransferToolWrongAnswer:
+        poll_individual_transfers = True
+
+    if poll_individual_transfers:
+        logger(logging.ERROR, 'Problem querying %s on %s. All jobs are being checked individually' % (list(transfers_by_eid), transfertool_obj))
+        for external_id, transfers in transfers_by_eid.items():
+            logger(logging.DEBUG, 'Checking %s on %s' % (external_id, transfertool_obj))
+            try:
+                _poll_transfers(transfertool_obj, {external_id: transfers}, timeout, logger)
+            except Exception as err:
+                logger(logging.ERROR, 'Problem querying %s on %s . Error returned : %s' % (external_id, transfertool_obj, str(err)))
+
+
+def _poll_transfers(transfertool_obj, transfers_by_eid, timeout, logger):
+    """
+    Helper function for poll_transfers which performs the actual polling and database update.
+    """
+    is_bulk = len(transfers_by_eid) > 1
+    try:
+        stopwatch = Stopwatch()
+        logger(logging.INFO, 'Polling %i transfers against %s with timeout %s' % (len(transfers_by_eid), transfertool_obj, timeout))
+        resps = transfertool_obj.bulk_query(requests_by_eid=transfers_by_eid, timeout=timeout)
+        stopwatch.stop()
+        METRICS.timer('bulk_query_transfers').observe(stopwatch.elapsed / (len(transfers_by_eid) or 1))
+        logger(logging.DEBUG, 'Polled %s transfer requests status in %s seconds' % (len(transfers_by_eid), stopwatch.elapsed))
+    except TransferToolTimeout as error:
+        logger(logging.ERROR, str(error))
+        return
+    except TransferToolWrongAnswer as error:
+        logger(logging.ERROR, str(error))
+        if is_bulk:
+            raise  # The calling context will retry transfers one-by-one
+        else:
+            return
+    except RequestException as error:
+        logger(logging.ERROR, "Failed to contact FTS server: %s" % (str(error)))
+        return
+    except Exception:
+        logger(logging.ERROR, "Failed to query FTS info", exc_info=True)
+        return
+
+    tss = time.time()
+    logger(logging.DEBUG, 'Updating %s transfer requests status' % (len(transfers_by_eid)))
+    cnt = 0
+
+    request_ids = set(itertools.chain.from_iterable(transfers_by_eid.values()))
+    for transfer_id in resps:
+        try:
+            transf_resp = resps[transfer_id]
+            # transf_resp is None: Lost.
+            #             is Exception: Failed to get fts job status.
+            #             is {}: No terminated jobs.
+            #             is {request_id: {file_status}}: terminated jobs.
+            if transf_resp is None:
+                for request_id, request in transfers_by_eid[transfer_id].items():
+                    transfer_core.mark_transfer_lost(request, logger=logger)
+                METRICS.counter('transfer_lost').inc()
+            elif isinstance(transf_resp, Exception):
+                logger(logging.WARNING, "Failed to poll FTS(%s) job (%s): %s" % (transfertool_obj, transfer_id, transf_resp))
+                METRICS.counter('query_transfer_exception').inc()
+            else:
+                for request_id in request_ids.intersection(transf_resp):
+                    ret = transfer_core.update_transfer_state(transf_resp[request_id], logger=logger)
+                    # if True, really update request content; if False, only touch request
+                    if ret:
+                        cnt += 1
+                    METRICS.counter('update_request_state.{updated}').labels(updated=ret).inc()
+
+            # should touch transfers.
+            # Otherwise if one bulk transfer includes many requests and one is not terminated, the transfer will be poll again.
+            transfer_core.touch_transfer(transfertool_obj.external_host, transfer_id)
+        except (DatabaseException, DatabaseError) as error:
+            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                logger(logging.WARNING, "Lock detected when handling request %s - skipping" % transfer_id)
+            else:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+    logger(logging.DEBUG, 'Finished updating %s transfer requests status (%i requests state changed) in %s seconds' % (len(transfers_by_eid), cnt, (time.time() - tss)))

rucio/daemons/conveyor/preparer.py

@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import threading
+from time import time
+from types import FrameType
+from typing import TYPE_CHECKING, Optional
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.config import config_get_list
+from rucio.common.exception import RucioException
+from rucio.common.logging import setup_logging
+from rucio.core import transfer as transfer_core
+from rucio.core.request import set_requests_state_if_possible, list_and_mark_transfer_requests_and_source_replicas
+from rucio.core.topology import Topology, ExpiringObjectCache
+from rucio.core.transfer import prepare_transfers, list_transfer_admin_accounts, build_transfer_paths, ProtocolFactory
+from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
+from rucio.db.sqla.constants import RequestState, RequestType
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+    from rucio.daemons.common import HeartbeatHandler
+
+GRACEFUL_STOP = threading.Event()
+DAEMON_NAME = 'conveyor-preparer'
+
+
+def stop(signum: Optional[int] = None, frame: Optional[FrameType] = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    GRACEFUL_STOP.set()
+
+
+def run(
+        once=False,
+        threads=1,
+        sleep_time=10,
+        bulk=100,
+        ignore_availability: bool = False
+):
+    """
+    Running the preparer daemon either once or by default in a loop until stop is called.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    cached_topology = ExpiringObjectCache(ttl=300, new_obj_fnc=lambda: Topology(ignore_availability=ignore_availability))
+
+    preparer(
+        once=once,
+        sleep_time=sleep_time,
+        bulk=bulk,
+        ignore_availability=ignore_availability,
+        cached_topology=cached_topology,
+        total_threads=threads
+    )
+
+
+def preparer(
+        once,
+        sleep_time: int = 10,
+        bulk: int = 100,
+        ignore_availability: bool = False,
+        partition_wait_time: int = 10,
+        transfertools=None,
+        cached_topology=None,
+        total_threads=1
+):
+    # Make an initial heartbeat so that all instanced daemons have the correct worker number on the next try
+    executable = DAEMON_NAME
+    if not transfertools:
+        transfertools = config_get_list('conveyor', 'transfertool', False, None)
+
+    @db_workqueue(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=executable,
+        partition_wait_time=partition_wait_time,
+        sleep_time=sleep_time)
+    def _db_producer(*, activity: str, heartbeat_handler: "HeartbeatHandler"):
+        return _fetch_requests(
+            bulk=bulk,
+            ignore_availability=ignore_availability,
+            cached_topology=cached_topology,
+            heartbeat_handler=heartbeat_handler,
+            set_last_processed_by=not once,
+        )
+
+    def _consumer(batch):
+        return _handle_requests(
+            batch,
+            transfertools=transfertools,
+            bulk=bulk,
+        )
+
+    ProducerConsumerDaemon(
+        producers=[_db_producer],
+        consumers=[_consumer for _ in range(total_threads)],
+        graceful_stop=GRACEFUL_STOP,
+    ).run()
+
+
+def _fetch_requests(
+        bulk: int,
+        ignore_availability: bool,
+        cached_topology,
+        heartbeat_handler: "HeartbeatHandler",
+        set_last_processed_by: bool,
+        *,
+        session: "Optional[Session]" = None,
+):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    topology = cached_topology.get() if cached_topology else Topology(ignore_availability=ignore_availability)
+    topology.configure_multihop(logger=logger, session=session)
+    requests_with_sources = list_and_mark_transfer_requests_and_source_replicas(
+        rse_collection=topology,
+        processed_by=heartbeat_handler.short_executable if set_last_processed_by else None,
+        total_workers=total_workers,
+        worker_number=worker_number,
+        limit=bulk,
+        request_state=RequestState.PREPARING,
+        request_type=[RequestType.TRANSFER, RequestType.STAGEIN],
+        ignore_availability=ignore_availability,
+        session=session,
+    )
+    must_sleep = False
+    if len(requests_with_sources) < bulk / 2:
+        logger(logging.INFO, "Only %s transfers, which is less than half of the bulk %s", len(requests_with_sources), bulk)
+        must_sleep = True
+    return must_sleep, (topology, requests_with_sources)
+
+
+def _handle_requests(
+        batch,
+        *,
+        transfertools: Optional[list[str]] = None,
+        bulk: int = 100,
+        logger=logging.log,
+):
+    topology, requests_with_sources = batch
+
+    if not transfertools:
+        transfertools = list(transfer_core.TRANSFERTOOL_CLASSES_BY_NAME)
+
+    start_time = time()
+    try:
+        admin_accounts = list_transfer_admin_accounts()
+
+        ret = build_transfer_paths(
+            topology=topology,
+            protocol_factory=ProtocolFactory(),
+            requests_with_sources=list(requests_with_sources.values()),
+            admin_accounts=admin_accounts,
+            preparer_mode=True,
+            logger=logger,
+        )
+        requests_handled = sum(len(i) for i in ret)
+        if not requests_handled:
+            updated_msg = 'had nothing to do'
+        else:
+            candidate_paths, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source, _ = ret
+            updated_reqs, reqs_no_transfertool = prepare_transfers(candidate_paths, transfertools=transfertools, logger=logger)
+            updated_msg = f'updated {len(updated_reqs)}/{bulk} requests'
+
+            if reqs_no_transfertool:
+                logger(logging.INFO, "Ignoring request because of unsupported transfertool: %s", reqs_no_transfertool)
+            reqs_no_source.update(reqs_no_transfertool)
+            if reqs_no_source:
+                logger(logging.INFO, "Marking requests as no-sources: %s", reqs_no_source)
+                set_requests_state_if_possible(reqs_no_source, RequestState.NO_SOURCES, logger=logger)
+            if reqs_only_tape_source:
+                logger(logging.INFO, "Marking requests as only-tape-sources: %s", reqs_only_tape_source)
+                set_requests_state_if_possible(reqs_only_tape_source, RequestState.ONLY_TAPE_SOURCES, logger=logger)
+            if reqs_scheme_mismatch:
+                logger(logging.INFO, "Marking requests as scheme-mismatch: %s", reqs_scheme_mismatch)
+                set_requests_state_if_possible(reqs_scheme_mismatch, RequestState.MISMATCH_SCHEME, logger=logger)
+    except RucioException:
+        logger(logging.ERROR, 'errored with a RucioException, retrying later', exc_info=True)
+        updated_msg = 'errored'
+    logger(logging.INFO, '%s, taking %.3f seconds' % (updated_msg, time() - start_time))