rucio 32.3.1__tar.gz → 32.4.0__tar.gz

{rucio-32.3.1 → rucio-32.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 32.3.1
+Version: 32.4.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-32.3.1 → rucio-32.4.0}/lib/rucio/transfertool/fts3.py

@@ -31,7 +31,7 @@ from requests.adapters import ReadTimeout
 from requests.packages.urllib3 import disable_warnings  # pylint: disable=import-error
 
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get, config_get_bool
+from rucio.common.config import config_get, config_get_bool, config_get_int
 from rucio.common.constants import FTS_JOB_TYPE, FTS_STATE, FTS_COMPLETE_STATE
 from rucio.common.exception import TransferToolTimeout, TransferToolWrongAnswer, DuplicateFileTransferSubmission
 from rucio.common.stopwatch import Stopwatch
@@ -104,6 +104,61 @@ PATH_CHECKSUM_VALIDATION_STRATEGY: dict[tuple[str, str], str] = {
     ('none', 'none'): 'none',
 }
 
+_SCITAGS_NEXT_REFRESH = datetime.datetime.utcnow()
+_SCITAGS_EXP_ID = None
+_SCITAGS_ACTIVITY_IDS = {}
+
+
+def _scitags_ids(logger: Callable[..., Any] = logging.log) -> "tuple[int | None, dict[str, int]]":
+    """
+    Re-fetch if needed and return the scitags ids
+    """
+    enabled = config_get_bool('packet-marking', 'enabled', default=False)
+    if not enabled:
+        return None, {}
+
+    now = datetime.datetime.utcnow()
+    global _SCITAGS_ACTIVITY_IDS
+    global _SCITAGS_EXP_ID
+    global _SCITAGS_NEXT_REFRESH
+    if _SCITAGS_NEXT_REFRESH < now:
+        exp_name = config_get('packet-marking', 'exp_name', default='')
+        fetch_url = config_get('packet-marking', 'fetch_url', default='https://www.scitags.org/api.json')
+        fetch_interval = config_get_int('packet-marking', 'fetch_interval', default=datetime.timedelta(hours=48).seconds)
+        fetch_timeout = config_get_int('packet-marking', 'fetch_timeout', default=5)
+
+        _SCITAGS_NEXT_REFRESH = now + datetime.timedelta(seconds=fetch_interval)
+
+        if exp_name:
+            had_exception = False
+            exp_id = None
+            activity_ids = {}
+            try:
+                result = requests.get(fetch_url, timeout=fetch_timeout)
+                if result and result.status_code == 200:
+                    marks = result.json()
+                    for experiment in marks.get('experiments', []):
+                        if experiment.get('expName') == exp_name:
+                            exp_id = experiment.get('expId')
+                            for activity_dict in experiment.get('activities', []):
+                                activity_name = activity_dict.get('activityName')
+                                activity_id = activity_dict.get('activityId')
+                                if activity_name and activity_id:
+                                    activity_ids[activity_name] = int(activity_id)
+                            break
+            except (requests.exceptions.RequestException, TypeError, ValueError):
+                had_exception = True
+                logger(logging.WARNING, 'Failed to fetch the scitags markings', exc_info=True)
+
+            if had_exception:
+                # Retry quicker after fetch errors
+                _SCITAGS_NEXT_REFRESH = min(_SCITAGS_NEXT_REFRESH, now + datetime.timedelta(minutes=5))
+            else:
+                _SCITAGS_EXP_ID = exp_id
+                _SCITAGS_ACTIVITY_IDS = activity_ids
+
+    return _SCITAGS_EXP_ID, _SCITAGS_ACTIVITY_IDS
+
 
 def _pick_cert_file(vo: "Optional[str]") -> "Optional[str]":
     cert = None
@@ -791,6 +846,8 @@ class FTS3Transfertool(Transfertool):
             self.cert = None
             self.verify = True  # True is the default setting of a requests.* method
 
+        self.scitags_exp_id, self.scitags_activity_ids = _scitags_ids(logger=logger)
+
     @classmethod
     def _pick_fts_servers(cls, source_rse: "RseData", dest_rse: "RseData"):
         """
@@ -882,6 +939,10 @@ class FTS3Transfertool(Transfertool):
             'selection_strategy': self.source_strategy if self.source_strategy else _configured_source_strategy(transfer.rws.activity, logger=self.logger),
             'activity': rws.activity
         }
+        if isinstance(self.scitags_exp_id, int):
+            activity_id = self.scitags_activity_ids.get(rws.activity)
+            if isinstance(activity_id, int):
+                t_file['metadata']['scitags_id'] = self.scitags_exp_id << 6 | activity_id
         return t_file
 
     def submit(self, transfers, job_params, timeout=None):

{rucio-32.3.1 → rucio-32.4.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '32.3.1',
+    'version': '32.4.0',
     'branch_nick': 'release-32-LTS',
-    'revision_id': 'ebd75561f6fe9f239acb0fcd59956951550a90fa',
-    'revno': 12625
+    'revision_id': 'b32edac1a7e6c5f13575c92112afaa9e4b7e31b8',
+    'revno': 12629
 }

{rucio-32.3.1 → rucio-32.4.0}/lib/rucio/web/rest/flaskapi/v1/dids.py

@@ -24,7 +24,7 @@ from rucio.api.did import add_did, add_dids, list_content, list_content_history,
     resurrect, get_users_following_did, remove_did_from_followed, add_did_to_followed, delete_metadata, \
     set_metadata_bulk, set_dids_metadata_bulk
 from rucio.api.rule import list_replication_rules, list_associated_replication_rules_for_file
-from rucio.common.exception import ScopeNotFound, DataIdentifierNotFound, DataIdentifierAlreadyExists, \
+from rucio.common.exception import ScopeNotFound, DatabaseException, DataIdentifierNotFound, DataIdentifierAlreadyExists, \
     DuplicateContent, AccessDenied, KeyNotFound, Duplicate, InvalidValueForKey, UnsupportedStatus, \
     UnsupportedOperation, RSENotFound, RuleNotFound, InvalidMetadata, InvalidPath, FileAlreadyExists, InvalidObject, FileConsistencyMismatch
 from rucio.common.utils import render_json, APIEncoder
@@ -655,6 +655,15 @@ class DIDs(ErrorHandlingMethodView):
             return generate_http_error_flask(409, error)
         except AccessDenied as error:
             return generate_http_error_flask(401, error)
+        except DatabaseException as error:
+            if 'DELETED_DIDS_PK violated' in str(error):
+                return generate_http_error_flask(
+                    status_code=406,
+                    exc=error.__class__.__name__,
+                    exc_msg=str('A deleted DID {} with scope {} is reused'.format(name, scope))
+                )
+            else:
+                return generate_http_error_flask(406, error)
 
         return 'Created', 201
 

{rucio-32.3.1 → rucio-32.4.0}/tests/test_clients.py

@@ -15,10 +15,7 @@
 
 from datetime import datetime, timedelta
 
-from http.server import SimpleHTTPRequestHandler
-from http.server import HTTPServer
 from os import rename
-from threading import Thread
 
 import pytest
 
@@ -27,6 +24,7 @@ from rucio.client.client import Client
 from rucio.common.config import config_get, config_set
 from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException
 from rucio.common.utils import execute
+from tests.mocks.mock_http_server import MockServer
 
 
 @pytest.fixture
@@ -37,43 +35,6 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path
     config_set('client', 'auth_token_file_path', str(tmp_path / f'{function_scope_prefix}token'))
 
 
-class MockServer:
-    """
-    Start A simple http server in a separate thread to serve as MOCK for testing the client
-    """
-
-    class Handler(SimpleHTTPRequestHandler):
-        def send_code_and_message(self, code, headers, message):
-            """
-            Helper which wraps the quite-low-level BaseHTTPRequestHandler primitives and is used to send reponses.
-            """
-            self.send_response(code)
-            self.send_header("Content-type", "text/plain")
-            for name, content in headers.items():
-                self.send_header(name, content)
-            self.end_headers()
-            self.wfile.write(message.encode())
-
-    def __init__(self, request_handler_cls):
-        self.server = HTTPServer(('localhost', 0), request_handler_cls)
-        self.thread = Thread(target=self.server.serve_forever)
-        self.thread.daemon = True
-
-    def __enter__(self):
-        self.thread.start()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.server.shutdown()
-        self.thread.join()
-        self.server.server_close()
-
-    @property
-    def base_url(self):
-        name, port = self.server.server_address
-        return 'http://{}:{}'.format(name, port)
-
-
 @pytest.mark.usefixtures("client_token_path_override")
 class TestBaseClient:
     """ To test Clients"""

{rucio-32.3.1 → rucio-32.4.0}/tests/test_conveyor.py

@@ -49,6 +49,7 @@ from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
 from rucio.transfertool.fts3 import FTS3Transfertool
 from tests.ruciopytest import NoParallelGroups
+from tests.mocks.mock_http_server import MockServer
 
 MAX_POLL_WAIT_SECONDS = 60
 TEST_FTS_HOST = 'https://fts:8446'
@@ -118,6 +119,24 @@ def __get_source(request_id, src_rse_id, scope, name, *, session=None):
         .first()
 
 
+@pytest.fixture
+def scitags_mock(core_config_mock):
+    """Run a mock http server which always returns the content of scitags.json from test/inputs"""
+    from tests.inputs import SCITAGS_JSON
+    from pathlib import Path
+
+    class _SendScitagsJson(MockServer.Handler):
+        def do_GET(self):
+            file_content = Path(SCITAGS_JSON).read_text()
+            self.send_code_and_message(200, {'Content-Type': 'application/json'}, file_content)
+
+    with MockServer(_SendScitagsJson) as mock_server:
+        core_config.set('packet-marking', 'enabled',  True)
+        core_config.set('packet-marking', 'fetch_url',  mock_server.base_url)
+        core_config.set('packet-marking', 'exp_name',  'atlas')
+        yield mock_server
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(groups=[NoParallelGroups.XRD, NoParallelGroups.SUBMITTER, NoParallelGroups.POLLER, NoParallelGroups.FINISHER])
@@ -528,7 +547,7 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
 ]}], indirect=True)
-def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock, metrics_mock):
+def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock, metrics_mock, scitags_mock):
     """
     Verify that the receiver correctly handles successful multihop jobs
     """
@@ -547,7 +566,7 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock
 
         did = did_factory.upload_test_file(src_rse)
         rule_priority = 5
-        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None, priority=rule_priority)
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None, priority=rule_priority, activity='test')
         submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_state_transition(dst_rse_id=jump_rse_id, run_poller=False, **did)
@@ -556,7 +575,9 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock
         assert request['state'] == RequestState.DONE
 
         fts_response = FTS3Transfertool(external_host=TEST_FTS_HOST).bulk_query({request['external_id']: {request['id']: request}})
-        assert fts_response[request['external_id']][request['id']].job_response['priority'] == rule_priority
+        fts_response = fts_response[request['external_id']][request['id']]
+        assert fts_response.job_response['priority'] == rule_priority
+        assert fts_response.file_response['file_metadata'].get('scitags_id') is not None
 
         # Two hops; both handled by receiver
         assert metrics_mock.get_sample_value('rucio_daemons_conveyor_receiver_update_request_state_total', labels={'updated': 'True'}) >= 2