rucio 32.3.0__tar.gz → 32.4.0__tar.gz

{rucio-32.3.0 → rucio-32.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 32.3.0
+Version: 32.4.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py

@@ -367,28 +367,30 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: Opt
                         file_metadata_datatype = str(file_metadata["datatype"])
                         file_metadata_scope = str(file_metadata["scope"])
                         action = ""
-                        for policy in json_data:
-                            match_scope = False
-                            match_datatype = False
-
-                            if not policy.get("scope", []):
-                                match_scope = True
-                            for scope in policy.get("scope", []):
-                                if re.match(scope, file_metadata_scope):
+                        if file_metadata_datatype:
+                            # Some files don't have a datatype. They should be ignored.
+                            for policy in json_data:
+                                match_scope = False
+                                match_datatype = False
+
+                                if not policy.get("scope", []):
                                     match_scope = True
-                                    break
+                                for scope in policy.get("scope", []):
+                                    if re.match(scope, file_metadata_scope):
+                                        match_scope = True
+                                        break
 
-                            if not policy.get("datatype", []):
-                                match_datatype = True
-                            for datatype in policy.get("datatype", []):
-                                if re.match(datatype, file_metadata_datatype):
+                                if not policy.get("datatype", []):
                                     match_datatype = True
-                                    break
+                                for datatype in policy.get("datatype", []):
+                                    if re.match(datatype, file_metadata_datatype):
+                                        match_datatype = True
+                                        break
 
-                            if match_scope and match_datatype:
-                                action = policy["action"]
-                                logger(logging.INFO, "The action that will be performed is %s", action)
-                                break
+                                if match_scope and match_datatype:
+                                    action = policy["action"]
+                                    logger(logging.INFO, "The action that will be performed is %s", action)
+                                    break
 
                         if not action:
                             logger(logging.WARNING, "No recognised actions (ignore/declare bad) found in policy file (etc/suspicious_replica_recoverer.json). Replica will be ignored by default.")

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/rse/protocols/webdav.py

@@ -483,7 +483,7 @@ class Default(protocol.RSEProtocol):
             :raises SourceNotFound: if the source file was not found on the referred storage.
             :raises RSEAccessDenied: in case of permission issue.
 
-            :returns: a dict with two keys, filesize and adler32 of the file provided in path.
+            :returns: a dict with filesize of the file provided in path as a key.
         """
         headers = {'Depth': '1'}
         dict_ = {}

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/transfertool/fts3.py

@@ -31,7 +31,7 @@ from requests.adapters import ReadTimeout
 from requests.packages.urllib3 import disable_warnings  # pylint: disable=import-error
 
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get, config_get_bool
+from rucio.common.config import config_get, config_get_bool, config_get_int
 from rucio.common.constants import FTS_JOB_TYPE, FTS_STATE, FTS_COMPLETE_STATE
 from rucio.common.exception import TransferToolTimeout, TransferToolWrongAnswer, DuplicateFileTransferSubmission
 from rucio.common.stopwatch import Stopwatch
@@ -104,6 +104,61 @@ PATH_CHECKSUM_VALIDATION_STRATEGY: dict[tuple[str, str], str] = {
     ('none', 'none'): 'none',
 }
 
+_SCITAGS_NEXT_REFRESH = datetime.datetime.utcnow()
+_SCITAGS_EXP_ID = None
+_SCITAGS_ACTIVITY_IDS = {}
+
+
+def _scitags_ids(logger: Callable[..., Any] = logging.log) -> "tuple[int | None, dict[str, int]]":
+    """
+    Re-fetch if needed and return the scitags ids
+    """
+    enabled = config_get_bool('packet-marking', 'enabled', default=False)
+    if not enabled:
+        return None, {}
+
+    now = datetime.datetime.utcnow()
+    global _SCITAGS_ACTIVITY_IDS
+    global _SCITAGS_EXP_ID
+    global _SCITAGS_NEXT_REFRESH
+    if _SCITAGS_NEXT_REFRESH < now:
+        exp_name = config_get('packet-marking', 'exp_name', default='')
+        fetch_url = config_get('packet-marking', 'fetch_url', default='https://www.scitags.org/api.json')
+        fetch_interval = config_get_int('packet-marking', 'fetch_interval', default=datetime.timedelta(hours=48).seconds)
+        fetch_timeout = config_get_int('packet-marking', 'fetch_timeout', default=5)
+
+        _SCITAGS_NEXT_REFRESH = now + datetime.timedelta(seconds=fetch_interval)
+
+        if exp_name:
+            had_exception = False
+            exp_id = None
+            activity_ids = {}
+            try:
+                result = requests.get(fetch_url, timeout=fetch_timeout)
+                if result and result.status_code == 200:
+                    marks = result.json()
+                    for experiment in marks.get('experiments', []):
+                        if experiment.get('expName') == exp_name:
+                            exp_id = experiment.get('expId')
+                            for activity_dict in experiment.get('activities', []):
+                                activity_name = activity_dict.get('activityName')
+                                activity_id = activity_dict.get('activityId')
+                                if activity_name and activity_id:
+                                    activity_ids[activity_name] = int(activity_id)
+                            break
+            except (requests.exceptions.RequestException, TypeError, ValueError):
+                had_exception = True
+                logger(logging.WARNING, 'Failed to fetch the scitags markings', exc_info=True)
+
+            if had_exception:
+                # Retry quicker after fetch errors
+                _SCITAGS_NEXT_REFRESH = min(_SCITAGS_NEXT_REFRESH, now + datetime.timedelta(minutes=5))
+            else:
+                _SCITAGS_EXP_ID = exp_id
+                _SCITAGS_ACTIVITY_IDS = activity_ids
+
+    return _SCITAGS_EXP_ID, _SCITAGS_ACTIVITY_IDS
+
 
 def _pick_cert_file(vo: "Optional[str]") -> "Optional[str]":
     cert = None
@@ -791,6 +846,8 @@ class FTS3Transfertool(Transfertool):
             self.cert = None
             self.verify = True  # True is the default setting of a requests.* method
 
+        self.scitags_exp_id, self.scitags_activity_ids = _scitags_ids(logger=logger)
+
     @classmethod
     def _pick_fts_servers(cls, source_rse: "RseData", dest_rse: "RseData"):
         """
@@ -882,6 +939,10 @@ class FTS3Transfertool(Transfertool):
             'selection_strategy': self.source_strategy if self.source_strategy else _configured_source_strategy(transfer.rws.activity, logger=self.logger),
             'activity': rws.activity
         }
+        if isinstance(self.scitags_exp_id, int):
+            activity_id = self.scitags_activity_ids.get(rws.activity)
+            if isinstance(activity_id, int):
+                t_file['metadata']['scitags_id'] = self.scitags_exp_id << 6 | activity_id
         return t_file
 
     def submit(self, transfers, job_params, timeout=None):

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '32.3.0',
+    'version': '32.4.0',
     'branch_nick': 'release-32-LTS',
-    'revision_id': '7a033c118633c83dd7d588b4188b457f77f5dedc',
-    'revno': 12618
+    'revision_id': 'b32edac1a7e6c5f13575c92112afaa9e4b7e31b8',
+    'revno': 12629
 }

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/web/rest/flaskapi/v1/dids.py

@@ -24,7 +24,7 @@ from rucio.api.did import add_did, add_dids, list_content, list_content_history,
     resurrect, get_users_following_did, remove_did_from_followed, add_did_to_followed, delete_metadata, \
     set_metadata_bulk, set_dids_metadata_bulk
 from rucio.api.rule import list_replication_rules, list_associated_replication_rules_for_file
-from rucio.common.exception import ScopeNotFound, DataIdentifierNotFound, DataIdentifierAlreadyExists, \
+from rucio.common.exception import ScopeNotFound, DatabaseException, DataIdentifierNotFound, DataIdentifierAlreadyExists, \
     DuplicateContent, AccessDenied, KeyNotFound, Duplicate, InvalidValueForKey, UnsupportedStatus, \
     UnsupportedOperation, RSENotFound, RuleNotFound, InvalidMetadata, InvalidPath, FileAlreadyExists, InvalidObject, FileConsistencyMismatch
 from rucio.common.utils import render_json, APIEncoder
@@ -655,6 +655,15 @@ class DIDs(ErrorHandlingMethodView):
             return generate_http_error_flask(409, error)
         except AccessDenied as error:
             return generate_http_error_flask(401, error)
+        except DatabaseException as error:
+            if 'DELETED_DIDS_PK violated' in str(error):
+                return generate_http_error_flask(
+                    status_code=406,
+                    exc=error.__class__.__name__,
+                    exc_msg=str('A deleted DID {} with scope {} is reused'.format(name, scope))
+                )
+            else:
+                return generate_http_error_flask(406, error)
 
         return 'Created', 201
 

{rucio-32.3.0 → rucio-32.4.0}/lib/rucio/web/rest/flaskapi/v1/rses.py

@@ -43,15 +43,12 @@ class RSEs(ErrorHandlingMethodView):
         description: Lists all RSEs.
         tags:
           - Rucio Storage Elements
-        requestBody:
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  expression:
-                    description: An RSE expression.
-                    type: string
+        parameters:
+        - name: expression
+          in: query
+          description: RSE expression to select RSEs.
+          schema:
+            type: string
         responses:
           200:
             description: OK

{rucio-32.3.0 → rucio-32.4.0}/tests/test_api_external_representation.py

@@ -36,7 +36,6 @@ from rucio.api.subscription import add_subscription, list_subscriptions, list_su
 from rucio.common.config import config_get_bool
 from rucio.common.types import InternalScope
 from rucio.common.utils import api_update_return_dict, generate_uuid
-from rucio.core.rse import get_rse_id
 from rucio.core.vo import add_vo, vo_exists
 from rucio.daemons.abacus import rse as abacus_rse
 from rucio.daemons.judge import cleaner
@@ -93,34 +92,31 @@ class TestApiExternalRepresentation:
     @classmethod
     def setUpClass(cls):
         # Get test RSEs
-        cls.rse_name = 'MOCK'
-        cls.rse_id = get_rse_id(rse=cls.rse_name, vo=cls.vo)
-        cls.rse2_name = 'MOCK2'
-        cls.rse2_id = get_rse_id(rse=cls.rse2_name, vo=cls.vo)
-
         cls.rse3_name = rse_name_generator()
         cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', vo=cls.vo2 if cls.vo2 else cls.vo)
         cls.rse4_name = rse_name_generator()
         cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', vo=cls.vo2 if cls.vo2 else cls.vo)
         api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, vo=cls.vo2 or cls.vo)
 
-    def test_api_update_return_dict(self, account, account_name, scope_name, scope):
+    def test_api_update_return_dict(self, rse_factory, account, account_name, scope_name, scope):
         """ API: Test the conversion of dictionaries to external representation """
+        rse1, rse1_id = rse_factory.make_rse()
+        rse2, rse2_id = rse_factory.make_rse()
         test_dict = {'account': account,
                      'scope': scope,
-                     'rse_expression': 'MOCK|MOCK2',
-                     'rse_id': self.rse_id,
-                     'src_rse_id': self.rse_id,
-                     'source_rse_id': self.rse_id,
-                     'dest_rse_id': self.rse_id,
-                     'destination_rse_id': self.rse_id}
+                     'rse_expression': f'{rse1}|{rse2}',
+                     'rse_id': rse1_id,
+                     'src_rse_id': rse1_id,
+                     'source_rse_id': rse1_id,
+                     'dest_rse_id': rse1_id,
+                     'destination_rse_id': rse1_id}
         value = api_update_return_dict(test_dict)
-        expected = {'account': account_name, 'scope': scope_name, 'rse_expression': 'MOCK|MOCK2',
-                    'rse_id': self.rse_id, 'rse': self.rse_name,
-                    'src_rse_id': self.rse_id, 'src_rse': self.rse_name,
-                    'source_rse_id': self.rse_id, 'source_rse': self.rse_name,
-                    'dest_rse_id': self.rse_id, 'dest_rse': self.rse_name,
-                    'destination_rse_id': self.rse_id, 'destination_rse': self.rse_name}
+        expected = {'account': account_name, 'scope': scope_name, 'rse_expression': f'{rse1}|{rse2}',
+                    'rse_id': rse1_id, 'rse': rse1,
+                    'src_rse_id': rse1_id, 'src_rse': rse1,
+                    'source_rse_id': rse1_id, 'source_rse': rse1,
+                    'dest_rse_id': rse1_id, 'dest_rse': rse1,
+                    'destination_rse_id': rse1_id, 'destination_rse': rse1}
         assert value == expected
 
     def test_api_account(self, vo, vo2, account, account_name):
@@ -134,20 +130,22 @@ class TestApiExternalRepresentation:
             assert account.internal not in out
         assert '@' not in ' '.join(out)
 
-    def test_api_account_limit(self, vo, vo2, account_name):
+    def test_api_account_limit(self, rse_factory, vo, vo2, account_name):
         """ ACCOUNT_LIMIT (API): Test external representation of account limits """
         # Add mock account limits
-        rse_expr = '{}|{}'.format(self.rse_name, self.rse2_name)
-        api_acc_lim.set_local_account_limit(account_name, self.rse_name, 10000, issuer='root', vo=vo)
+        rse1, rse1_id = rse_factory.make_rse()
+        rse2, rse2_id = rse_factory.make_rse()
+        rse_expr = f'{rse1}|{rse2}'
+        api_acc_lim.set_local_account_limit(account_name, rse1, 10000, issuer='root', vo=vo)
         api_acc_lim.set_global_account_limit(account_name, rse_expr, 20000, issuer='root', vo=vo)
 
         out = api_acc_lim.get_local_account_limits(account_name, vo=vo)
-        assert self.rse_name in out
-        assert self.rse_id not in out
+        assert rse1 in out
+        assert rse1_id not in out
 
-        out = api_acc_lim.get_local_account_limit(account_name, self.rse_name, vo=vo)
-        assert self.rse_name in out
-        assert self.rse_id not in out
+        out = api_acc_lim.get_local_account_limit(account_name, rse1, vo=vo)
+        assert rse1 in out
+        assert rse1_id not in out
 
         out = api_acc_lim.get_global_account_limits(account_name, vo=vo)
         assert rse_expr in out
@@ -159,15 +157,15 @@ class TestApiExternalRepresentation:
         if vo2:
             assert 'vo={}&({})'.format(vo, rse_expr) not in out
 
-        out = api_acc_lim.get_local_account_usage(account_name, self.rse_name, issuer='root', vo=vo)
+        out = api_acc_lim.get_local_account_usage(account_name, rse1, issuer='root', vo=vo)
         out = list(out)
         assert 0 != len(out)
-        assert self.rse_id in [usage['rse_id'] for usage in out if 'rse_id' in usage]
+        assert rse1_id in [usage['rse_id'] for usage in out if 'rse_id' in usage]
         for usage in out:
             if 'rse_id' in usage:
                 assert 'rse' in usage
-                if usage['rse_id'] == self.rse_id:
-                    assert self.rse_name == usage["rse"]
+                if usage['rse_id'] == rse1_id:
+                    assert rse1 == usage["rse"]
 
         out = api_acc_lim.get_global_account_usage(account_name, rse_expr, issuer='root', vo=vo)
         out = list(out)
@@ -205,9 +203,11 @@ class TestApiExternalRepresentation:
         for user in out:
             assert user['user'] == account_name
 
-    def test_api_exporter(self, vo, vo2):
+    def test_api_exporter(self, vo, rse_factory, vo2):
         """ EXPORTER (API): Test external representation of exported data """
 
+        rse1, rse1_id = rse_factory.make_rse()
+        rse2, rse2_id = rse_factory.make_rse()
         out = export_data('root', vo=vo2 if vo2 else vo)
         rses = out['rses']
         assert self.rse3_name in rses
@@ -221,15 +221,15 @@ class TestApiExternalRepresentation:
 
         # check for interference from other VOs
         if vo2:
-            assert self.rse_name not in rses
-            assert self.rse_id not in rses
-            assert self.rse2_name not in rses
-            assert self.rse2_id not in rses
+            assert rse1 not in rses
+            assert rse1_id not in rses
+            assert rse2 not in rses
+            assert rse2_id not in rses
 
-            assert self.rse_name not in distances
-            assert self.rse_id not in distances
-            assert self.rse2_name not in distances
-            assert self.rse2_id not in distances
+            assert rse1 not in distances
+            assert rse1_id not in distances
+            assert rse2 not in distances
+            assert rse2_id not in distances
 
     def test_api_identity(self, vo, vo2, account, account_name):
         """ IDENTITY (API): Test external representation of identity accounts """
@@ -243,20 +243,26 @@ class TestApiExternalRepresentation:
         if vo2:
             assert account.internal not in out
 
-    def test_api_replica(self, vo, vo2, account_name, scope_name, scope):
+    def test_api_replica(self, vo, rse_factory, vo2, account_name, scope_name, scope):
         """ REPLICA (API): Test external representation of replicas """
 
         did = did_name_generator('file')
         did_parent = did_name_generator('dataset')
-        pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (scope_name, generate_uuid())
-        add_replicas(self.rse2_name, files=[{'scope': scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', vo=vo)
+        rse2, rse2_id = rse_factory.make_rse(scheme='srm', protocol_impl='rucio.rse.protocols.gfal.Default', deterministic=False)
+        protocols = api_rse.get_rse_protocols(rse2, issuer='root', vo=vo)
+        pfn = 'srm://%s:%s/srm/managerv2?SFN=%s%s/%s' % (protocols['protocols'][0]['hostname'],
+                                                         protocols['protocols'][0]['port'],
+                                                         protocols['protocols'][0]['prefix'],
+                                                         scope_name,
+                                                         generate_uuid())
+        add_replicas(rse2, files=[{'scope': scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', vo=vo)
 
         add_did(scope_name, did_parent, 'dataset', issuer='root', account=account_name, vo=vo)
         attachment = {'scope': scope_name, 'name': did_parent,
                       'dids': [{'scope': scope_name, 'name': did}]}
         attach_dids_to_dids([attachment], issuer='root', vo=vo)
 
-        out = get_did_from_pfns([pfn], self.rse2_name, vo=vo)
+        out = get_did_from_pfns([pfn], rse2, vo=vo)
         out = list(out)
         assert 0 != len(out)
         did_found = False
@@ -281,15 +287,17 @@ class TestApiExternalRepresentation:
                         assert scope.internal not in parent
         assert parents_found
 
-    def test_api_request(self, vo, account_name, scope_name):
+    def test_api_request(self, vo, rse_factory, account_name, scope_name):
         """ REQUEST (API): Test external representation of requests """
 
+        rse1, rse1_id = rse_factory.make_rse()
+        rse2, rse2_id = rse_factory.make_rse()
         did = did_name_generator('dataset')
-        add_did(scope_name, did, 'dataset', issuer='root', account=account_name, rse=self.rse_name, vo=vo)
+        add_did(scope_name, did, 'dataset', issuer='root', account=account_name, rse=rse1, vo=vo)
 
         requests = [{
-            'dest_rse_id': self.rse2_id,
-            'source_rse_id': self.rse_id,
+            'dest_rse_id': rse2_id,
+            'source_rse_id': rse1_id,
             'request_type': constants.RequestType.TRANSFER,
             'request_id': generate_uuid(),
             'name': did,
@@ -312,16 +320,16 @@ class TestApiExternalRepresentation:
         for r in reqs:
             assert r['scope'] == scope_name
             assert r['account'] == account_name
-            assert r['source_rse'] == self.rse_name
-            assert r['dest_rse'] == self.rse2_name
+            assert r['source_rse'] == rse1
+            assert r['dest_rse'] == rse2
 
-        out = get_request_by_did(scope_name, did, self.rse2_name, issuer='root', vo=vo)
+        out = get_request_by_did(scope_name, did, rse2, issuer='root', vo=vo)
         assert out['scope'] == scope_name
         assert out['account'] == account_name
-        assert out['dest_rse'] == self.rse2_name
-        assert out['source_rse'] == self.rse_name
+        assert out['dest_rse'] == rse2
+        assert out['source_rse'] == rse1
 
-        out = list_requests([self.rse_name], [self.rse2_name], [constants.RequestState.QUEUED], issuer='root', vo=vo)
+        out = list_requests([rse1], [rse2], [constants.RequestState.QUEUED], issuer='root', vo=vo)
         out = list(out)
         assert 0 != len(out)
         assert scope_name in [req['scope'] for req in out]
@@ -329,16 +337,18 @@ class TestApiExternalRepresentation:
             if req['scope'] == scope_name:
                 assert req['scope'] == scope_name
                 assert req['account'] == account_name
-                assert req['dest_rse'] == self.rse2_name
-                assert req['source_rse'] == self.rse_name
+                assert req['dest_rse'] == rse2
+                assert req['source_rse'] == rse1
 
     @pytest.mark.noparallel(reason='runs the reaper on a pre-defined rse, might interfere with other tests')
-    def test_api_rse(self, vo, vo2, account, account_name, scope_name):
+    def test_api_rse(self, vo, rse_factory, vo2, account, account_name, scope_name):
         """ RSE (API): Test external representation of RSEs """
 
-        out = api_rse.get_rse(self.rse_name, vo=vo)
-        assert out['rse'] == self.rse_name
-        assert out['id'] == self.rse_id
+        rse1, rse1_id = rse_factory.make_rse()
+        rse2, rse2_id = rse_factory.make_rse()
+        out = api_rse.get_rse(rse1, vo=vo)
+        assert out['rse'] == rse1
+        assert out['id'] == rse1_id
 
         out = api_rse.list_rses(vo=vo2 if vo2 else vo)
         out = list(out)
@@ -354,18 +364,18 @@ class TestApiExternalRepresentation:
                 assert rse['rse'] == self.rse4_name
 
         key = "KEY_" + generate_uuid()
-        api_rse.add_rse_attribute(self.rse_name, key, 1, issuer='root', vo=vo)
+        api_rse.add_rse_attribute(rse1, key, 1, issuer='root', vo=vo)
         out = api_rse.get_rses_with_attribute(key)
         out = list(out)
         assert 0 != len(out)
         for rse in out:
-            assert rse['rse'] == self.rse_name
+            assert rse['rse'] == rse1
 
-        out = api_rse.get_rse_protocols(self.rse_name, issuer='root', vo=vo)
-        assert out['rse'] == self.rse_name
+        out = api_rse.get_rse_protocols(rse1, issuer='root', vo=vo)
+        assert out['rse'] == rse1
 
         # add some account and RSE counters
-        rse_mock, rse_mock_id = self.rse_factory.make_mock_rse()
+        rse_mock, rse_mock_id = rse_factory.make_mock_rse()
         account_counter.del_counter(rse_id=rse_mock_id, account=account)
         account_counter.add_counter(rse_id=rse_mock_id, account=account)
         account_counter.increase(rse_id=rse_mock_id, account=account, files=1, bytes_=10)
@@ -393,11 +403,11 @@ class TestApiExternalRepresentation:
             reaper.run(once=True, include_rses=rse_mock, greedy=True)
         abacus_rse.run(once=True)
 
-        out = api_rse.parse_rse_expression('%s|%s' % (self.rse_name, self.rse2_name), vo=vo)
-        assert self.rse_name in out
-        assert self.rse2_name in out
-        assert self.rse_id not in out
-        assert self.rse2_id not in out
+        out = api_rse.parse_rse_expression(f'{rse1}|{rse2}', vo=vo)
+        assert rse1 in out
+        assert rse2 in out
+        assert rse1_id not in out
+        assert rse2_id not in out
 
     def test_api_scope(self, vo, vo2, account_name, scope_name, scope):
         """ SCOPE (API): Test external representation of scopes """

{rucio-32.3.0 → rucio-32.4.0}/tests/test_clients.py

@@ -15,10 +15,7 @@
 
 from datetime import datetime, timedelta
 
-from http.server import SimpleHTTPRequestHandler
-from http.server import HTTPServer
 from os import rename
-from threading import Thread
 
 import pytest
 
@@ -27,6 +24,7 @@ from rucio.client.client import Client
 from rucio.common.config import config_get, config_set
 from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException
 from rucio.common.utils import execute
+from tests.mocks.mock_http_server import MockServer
 
 
 @pytest.fixture
@@ -37,43 +35,6 @@ def client_token_path_override(file_config_mock, function_scope_prefix, tmp_path
     config_set('client', 'auth_token_file_path', str(tmp_path / f'{function_scope_prefix}token'))
 
 
-class MockServer:
-    """
-    Start A simple http server in a separate thread to serve as MOCK for testing the client
-    """
-
-    class Handler(SimpleHTTPRequestHandler):
-        def send_code_and_message(self, code, headers, message):
-            """
-            Helper which wraps the quite-low-level BaseHTTPRequestHandler primitives and is used to send reponses.
-            """
-            self.send_response(code)
-            self.send_header("Content-type", "text/plain")
-            for name, content in headers.items():
-                self.send_header(name, content)
-            self.end_headers()
-            self.wfile.write(message.encode())
-
-    def __init__(self, request_handler_cls):
-        self.server = HTTPServer(('localhost', 0), request_handler_cls)
-        self.thread = Thread(target=self.server.serve_forever)
-        self.thread.daemon = True
-
-    def __enter__(self):
-        self.thread.start()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.server.shutdown()
-        self.thread.join()
-        self.server.server_close()
-
-    @property
-    def base_url(self):
-        name, port = self.server.server_address
-        return 'http://{}:{}'.format(name, port)
-
-
 @pytest.mark.usefixtures("client_token_path_override")
 class TestBaseClient:
     """ To test Clients"""

{rucio-32.3.0 → rucio-32.4.0}/tests/test_conveyor.py

@@ -49,6 +49,7 @@ from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
 from rucio.transfertool.fts3 import FTS3Transfertool
 from tests.ruciopytest import NoParallelGroups
+from tests.mocks.mock_http_server import MockServer
 
 MAX_POLL_WAIT_SECONDS = 60
 TEST_FTS_HOST = 'https://fts:8446'
@@ -118,6 +119,24 @@ def __get_source(request_id, src_rse_id, scope, name, *, session=None):
         .first()
 
 
+@pytest.fixture
+def scitags_mock(core_config_mock):
+    """Run a mock http server which always returns the content of scitags.json from test/inputs"""
+    from tests.inputs import SCITAGS_JSON
+    from pathlib import Path
+
+    class _SendScitagsJson(MockServer.Handler):
+        def do_GET(self):
+            file_content = Path(SCITAGS_JSON).read_text()
+            self.send_code_and_message(200, {'Content-Type': 'application/json'}, file_content)
+
+    with MockServer(_SendScitagsJson) as mock_server:
+        core_config.set('packet-marking', 'enabled',  True)
+        core_config.set('packet-marking', 'fetch_url',  mock_server.base_url)
+        core_config.set('packet-marking', 'exp_name',  'atlas')
+        yield mock_server
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(groups=[NoParallelGroups.XRD, NoParallelGroups.SUBMITTER, NoParallelGroups.POLLER, NoParallelGroups.FINISHER])
@@ -528,7 +547,7 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
 ]}], indirect=True)
-def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock, metrics_mock):
+def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock, metrics_mock, scitags_mock):
     """
     Verify that the receiver correctly handles successful multihop jobs
     """
@@ -547,7 +566,7 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock
 
         did = did_factory.upload_test_file(src_rse)
         rule_priority = 5
-        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None, priority=rule_priority)
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None, priority=rule_priority, activity='test')
         submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_state_transition(dst_rse_id=jump_rse_id, run_poller=False, **did)
@@ -556,7 +575,9 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock
         assert request['state'] == RequestState.DONE
 
         fts_response = FTS3Transfertool(external_host=TEST_FTS_HOST).bulk_query({request['external_id']: {request['id']: request}})
-        assert fts_response[request['external_id']][request['id']].job_response['priority'] == rule_priority
+        fts_response = fts_response[request['external_id']][request['id']]
+        assert fts_response.job_response['priority'] == rule_priority
+        assert fts_response.file_response['file_metadata'].get('scitags_id') is not None
 
         # Two hops; both handled by receiver
         assert metrics_mock.get_sample_value('rucio_daemons_conveyor_receiver_update_request_state_total', labels={'updated': 'True'}) >= 2