rucio-clients 33.2.1__tar.gz → 33.4.0__tar.gz

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 33.2.1
+Version: 33.4.0
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/lib/rucio/client/diracclient.py

@@ -28,7 +28,7 @@ class DiracClient(BaseClient):
 
     DIRAC_BASEURL = 'dirac'
 
-    def add_files(self, lfns, ignore_availability=False):
+    def add_files(self, lfns, ignore_availability=False, parents_metadata=None):
         """
         Bulk add files :
         - Create the file and replica.
@@ -37,10 +37,11 @@ class DiracClient(BaseClient):
 
         :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
         :param ignore_availability: A boolean to ignore blocked sites.
+        :param parents_metadata: Metadata for selected hierarchy DIDs. (dictionary {'lpn': {key : value}}). Default=None
         """
         path = '/'.join([self.DIRAC_BASEURL, 'addfiles'])
         url = build_url(choice(self.list_hosts), path=path)
-        r = self._send_request(url, type_='POST', data=dumps({'lfns': lfns, 'ignore_availability': ignore_availability}))
+        r = self._send_request(url, type_='POST', data=dumps({'lfns': lfns, 'ignore_availability': ignore_availability, 'parents_metadata': parents_metadata}))
         if r.status_code == codes.created:
             return True
         else:

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/lib/rucio/client/subscriptionclient.py

@@ -95,9 +95,10 @@ class SubscriptionClient(BaseClient):
         result = self._send_request(url, type_='GET')
         if result.status_code == codes.ok:   # pylint: disable=no-member
             return self._load_json_data(result)
-        else:
-            exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
-            raise exc_cls(exc_msg)
+        if result.status_code == codes.not_found:
+            return []
+        exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
+        raise exc_cls(exc_msg)
 
     def update_subscription(self, name, account=None, filter_=None, replication_rules=None, comments=None, lifetime=None, retroactive=None, dry_run=None, priority=None):
         """

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/lib/rucio/rse/protocols/xrootd.py

@@ -24,6 +24,13 @@ from rucio.rse.protocols import protocol
 class Default(protocol.RSEProtocol):
     """ Implementing access to RSEs using the XRootD protocol using GSI authentication."""
 
+    @property
+    def _auth_env(self):
+        if self.auth_token:
+            return f"XrdSecPROTOCOL=ztn BEARER_TOKEN='{self.auth_token}'"
+        else:
+            return 'XrdSecPROTOCOL=gsi'
+
     def __init__(self, protocol_attr, rse_settings, logger=logging.log):
         """ Initializes the object with information about the referred RSE.
 
@@ -66,7 +73,7 @@ class Default(protocol.RSEProtocol):
         self.logger(logging.DEBUG, 'xrootd.exists: pfn: {}'.format(pfn))
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.exists: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -86,7 +93,7 @@ class Default(protocol.RSEProtocol):
 
         :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
-        self.logger(logging.DEBUG, 'xrootd.stat: path: {}'.format(path))
+        self.logger(logging.DEBUG, f'xrootd.stat: path: {path}')
         ret = {}
         chsum = None
         if path.startswith('root:'):
@@ -94,7 +101,7 @@ class Default(protocol.RSEProtocol):
 
         try:
             # xrdfs stat for getting filesize
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: filesize cmd: {}'.format(cmd))
             status_stat, out, err = execute(cmd)
             if status_stat == 0:
@@ -106,7 +113,7 @@ class Default(protocol.RSEProtocol):
                             break
 
             # xrdfs query checksum for getting checksum
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s query checksum %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} query checksum {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: checksum cmd: {}'.format(cmd))
             status_query, out, err = execute(cmd)
             if status_query == 0:
@@ -183,7 +190,7 @@ class Default(protocol.RSEProtocol):
         try:
             # The query stats call is not implemented on some xroot doors.
             # Workaround: fail, if server does not reply within 10 seconds for static config query
-            cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
+            cmd = f'{self._auth_env} XRD_REQUESTTIMEOUT=10 xrdfs {self.hostname}:{self.port} query config {self.hostname}:{self.port}'
             self.logger(logging.DEBUG, 'xrootd.connect: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -206,7 +213,7 @@ class Default(protocol.RSEProtocol):
         """
         self.logger(logging.DEBUG, 'xrootd.get: pfn: {}'.format(pfn))
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
+            cmd = f'{self._auth_env} xrdcp -f {pfn} {dest}'
             self.logger(logging.DEBUG, 'xrootd.get: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status == 54:
@@ -237,7 +244,7 @@ class Default(protocol.RSEProtocol):
         if not os.path.exists(source_url):
             raise exception.SourceNotFound()
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
+            cmd = f'{self._auth_env} xrdcp -f {source_url} {path}'
             self.logger(logging.DEBUG, 'xrootd.put: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -259,7 +266,7 @@ class Default(protocol.RSEProtocol):
             raise exception.SourceNotFound()
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} rm {path}'
             self.logger(logging.DEBUG, 'xrootd.delete: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -283,10 +290,10 @@ class Default(protocol.RSEProtocol):
             path = self.pfn2path(pfn)
             new_path = self.pfn2path(new_pfn)
             new_dir = new_path[:new_path.rindex('/') + 1]
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mkdir -p {new_dir}'
             self.logger(logging.DEBUG, 'xrootd.stat: mkdir cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mv {path} {new_path}'
             self.logger(logging.DEBUG, 'xrootd.stat: rename cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '33.2.1',
+    'version': '33.4.0',
     'branch_nick': 'release-33',
-    'revision_id': '40778f689d8c42f91d2e54f30c09aa7fef393fd7',
-    'revno': 12685
+    'revision_id': '501f39cf4ba83caccda9a683e1f4c6008ed1b685',
+    'revno': 12708
 }

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_belleii.py

@@ -85,6 +85,39 @@ def test_dirac_addfile(rse_factory, did_factory, root_account, did_client, dirac
         assert rules[0]['rse_expression'] == 'ANY=true'
         assert (rules[0]['expires_at'] - datetime.utcnow()).days == 30
 
+@skip_non_belleii
+def test_dirac_addfile_with_parents_meta(rse_factory, did_factory, root_account, did_client, dirac_client, rse_client, replica_client):
+    """ DIRAC (CLIENT): Test the functionality of the addfile method """
+    rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
+    rse_client.add_rse_attribute(rse=rse1, key='ANY', value='True')
+    config_set('dirac', 'lifetime', '{"user.*": 2592400}')
+    lfn_name = did_name_generator('file')
+    lfn_meta = {'events': 10, 'key1':'value1'}
+    # Create replicas on rse1 using addfile in mock scope (not lifetime)
+    lfns = [{'lfn': lfn_name, 'rse': rse1, 'bytes': 1, 'adler32': '0cc737eb', 'guid': generate_uuid(), 'meta': lfn_meta}]
+    files = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn']} for lfn in lfns]
+    reps = [{'scope': extract_scope(lfn['lfn'], [])[0], 'name': lfn['lfn'], 'rse': rse1} for lfn in lfns]
+    dataset = "/".join(lfns[0]['lfn'].split('/')[:-1])
+    container = "/".join(lfns[0]['lfn'].split('/')[:-2])
+    dataset_meta = {'project': 'data13_hip', 'run_number': 300000, 'mykey': 'myvalue'}
+    container_meta = {'containerkey': 'containervalue'}
+    parents_metadata = {dataset: dataset_meta, container: container_meta}
+    dirac_client.add_files(lfns=lfns, ignore_availability=False, parents_metadata=parents_metadata)
+    replicas = [rep for rep in replica_client.list_replicas(dids=files)]
+    for replica in replicas:
+        assert {'scope': replica['scope'], 'name': replica['name'], 'rse': list(replica['rses'].keys())[0]} in reps
+    
+    # check if metadata if properly created for file and parents
+    for lfn in lfns:
+        scope, name = extract_scope(lfn['lfn'], [])
+        metadata = did_client.get_metadata(scope, name, plugin='ALL')
+        assert all(item in metadata.items() for item in lfn_meta.items())
+        dsn_scope, dsn_name = extract_scope(dataset, [])
+        metadata = did_client.get_metadata(dsn_scope, dsn_name, plugin='ALL')
+        assert all(item in metadata.items() for item in dataset_meta.items())
+        con_scope, con_name = extract_scope(container, [])
+        metadata = did_client.get_metadata(con_scope , con_name, plugin='ALL')
+        assert all(item in metadata.items() for item in container_meta.items())
 
 @skip_non_belleii
 def test_belle2_schema(rse_factory, did_factory, root_account, did_client):

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_conveyor.py

@@ -52,7 +52,7 @@ from rucio.transfertool.fts3 import FTS3Transfertool
 from tests.ruciopytest import NoParallelGroups
 from tests.mocks.mock_http_server import MockServer
 
-MAX_POLL_WAIT_SECONDS = 60
+MAX_POLL_WAIT_SECONDS = 100
 TEST_FTS_HOST = 'https://fts:8446'
 
 
@@ -823,7 +823,6 @@ def test_non_deterministic_dst(did_factory, did_client, root_account, vo, caches
 
 
 @skip_rse_tests_with_accounts
-@pytest.mark.dirty(reason="Submits a transfer to FTS which cannot be ever executed in this test env.")
 @pytest.mark.noparallel(groups=[NoParallelGroups.STAGER, NoParallelGroups.POLLER, NoParallelGroups.FINISHER])
 def test_stager(rse_factory, did_factory, root_account, replica_client):
     """
@@ -861,8 +860,8 @@ def test_stager(rse_factory, did_factory, root_account, replica_client):
                                            'requested_at': datetime.utcnow()}])
     stager(once=True, rses=[{'id': rse_id} for rse_id in all_rses])
 
-    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
-    assert request['state'] == RequestState.SUBMITTED
+    replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, max_wait_seconds=2 * MAX_POLL_WAIT_SECONDS, **did)
+    assert replica['state'] == ReplicaState.AVAILABLE
 
 
 @pytest.mark.noparallel(groups=[NoParallelGroups.SUBMITTER, NoParallelGroups.FINISHER])
@@ -1579,6 +1578,53 @@ def test_checksum_validation(rse_factory, did_factory, root_account):
     assert request['state'] == RequestState.FAILED
 
 
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(groups=[NoParallelGroups.XRD, NoParallelGroups.SUBMITTER, NoParallelGroups.RECEIVER])
+@pytest.mark.parametrize("file_config_mock", [
+    {"overrides": [('oidc', 'admin_issuer', 'indigoiam')]},
+], indirect=True)
+def test_transfer_with_tokens(vo, did_factory, root_account, caches_mock, file_config_mock):
+    src_rse = 'WEB1'
+    src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+    dst_rse = 'XRD5'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    did = did_factory.upload_test_file(src_rse)
+
+    rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+
+    received_messages = {}
+
+    class ReceiverWrapper(Receiver):
+        """
+        Wrap receiver to record the last handled message for each given request_id
+        """
+        def _perform_request_update(self, msg, *, session=None, logger=logging.log):
+            ret = super()._perform_request_update(msg, session=session, logger=logger)
+            received_messages[msg['file_metadata']['request_id']] = msg
+            return ret
+
+    with patch('rucio.daemons.conveyor.receiver.Receiver', ReceiverWrapper):
+        receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1})
+        receiver_thread.start()
+        try:
+            submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
+            # Wait for the reception of the FTS Completion message for the submitted request
+            request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+            for i in range(MAX_POLL_WAIT_SECONDS):
+                if request['id'] in received_messages:
+                    break
+                if i == MAX_POLL_WAIT_SECONDS - 1:
+                    assert False  # Waited too long; fail the test
+                time.sleep(1)
+            assert received_messages[request['id']]['job_metadata']['auth_method'] == 'oauth2'
+        finally:
+            receiver_graceful_stop.set()
+            receiver_thread.join(timeout=5)
+            receiver_graceful_stop.clear()
+
+
 @pytest.mark.noparallel(groups=[NoParallelGroups.PREPARER])
 @pytest.mark.parametrize("file_config_mock", [{
     "overrides": [('conveyor', 'use_preparer', 'true')]

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_import_export.py

@@ -81,13 +81,6 @@ def reset_rses():
     db_session.commit()
 
 
-def test_active():
-    db_session = session.get_session()
-    if db_session.bind.dialect.name == 'sqlite':
-        return False
-    return True
-
-
 @pytest.fixture
 def importer_example_data(vo):
     if not config_has_section('importer'):

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_reaper.py

@@ -36,7 +36,8 @@ from rucio.daemons.reaper.dark_reaper import reaper as dark_reaper
 from rucio.daemons.reaper.reaper import run as run_reaper
 from rucio.db.sqla.models import ConstituentAssociationHistory
 from rucio.db.sqla.session import read_session
-from rucio.tests.common import rse_name_generator
+from rucio.tests.common import rse_name_generator, skip_rse_tests_with_accounts
+from tests.ruciopytest import NoParallelGroups
 
 __mock_protocol = {'scheme': 'MOCK',
                    'hostname': 'localhost',
@@ -481,3 +482,21 @@ def test_reaper_without_rse_usage(vo, caches_mock):
     cache_region.invalidate()
     reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, chunk_size=1000, scheme='MOCK')
     assert len(list(replica_core.list_replicas(dids, rse_expression=rse_name))) == nb_files - nb_epoch_tombstone
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(groups=[NoParallelGroups.WEB])
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.daemons.reaper.reaper.REGION'
+]}], indirect=True)
+@pytest.mark.parametrize("file_config_mock", [
+    {"overrides": [('oidc', 'admin_issuer', 'indigoiam')]},
+], indirect=True)
+def test_deletion_with_tokens(vo, did_factory, root_account, caches_mock, file_config_mock):
+    rse_name = 'WEB1'
+    did = did_factory.upload_test_file(rse_name)
+    for rule in list(rule_core.list_associated_rules_for_file(**did)):
+        rule_core.delete_rule(rule['id'])
+
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, chunk_size=1000, greedy=True, scheme='davs')

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_tpc.py

@@ -88,7 +88,7 @@ def test_tpc(containerized_rses, root_account, test_scope, did_factory, rse_clie
     paths, *_ = build_transfer_paths(topology=topology, protocol_factory=ProtocolFactory(), requests_with_sources=requests)
     [[_, [transfer_path]]] = paths.items()
     assert transfer_path[0].rws.rule_id == rule_id[0]
-    src_url = transfer_path[0].legacy_sources[0][1]
+    src_url = transfer_path[0].source_url(transfer_path[0].sources[0])
     dest_url = transfer_path[0].dest_url
     check_url(src_url, rse1_hostname, test_file_expected_pfn)
     check_url(dest_url, rse2_hostname, test_file_expected_pfn)

{rucio-clients-33.2.1 → rucio-clients-33.4.0}/tests/test_transfer.py

@@ -181,16 +181,16 @@ def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope, file_confi
     # On equal priority and distance, disk should be preferred over tape. Both disk sources will be returned
     [[_, [transfer]]] = pick_and_prepare_submission_path(topology=topology, protocol_factory=ProtocolFactory(),
                                                          requests_with_sources=requests).items()
-    assert len(transfer[0].legacy_sources) == 2
-    assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name)
+    assert len(transfer[0].sources) == 2
+    assert transfer[0].sources[0].rse.name in (disk1_rse_name, disk2_rse_name)
 
     # Change the rating of the disk RSEs. Disk still preferred, because it must fail twice before tape is tried
     disk1_source.ranking = -1
     disk2_source.ranking = -1
     [[_, [transfer]]] = pick_and_prepare_submission_path(topology=topology, protocol_factory=ProtocolFactory(),
                                                          requests_with_sources=requests).items()
-    assert len(transfer[0].legacy_sources) == 2
-    assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name)
+    assert len(transfer[0].sources) == 2
+    assert transfer[0].sources[0].rse.name in (disk1_rse_name, disk2_rse_name)
 
     # Change the rating of the disk RSEs again. Tape RSEs must now be preferred.
     # Multiple tape sources are not allowed. Only one tape RSE source must be returned.
@@ -200,21 +200,21 @@ def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope, file_confi
                                                         requests_with_sources=requests).items()
     assert len(transfers) == 1
     transfer = transfers[0]
-    assert len(transfer[0].legacy_sources) == 1
-    assert transfer[0].legacy_sources[0][0] in (tape1_rse_name, tape2_rse_name)
+    assert len(transfer[0].sources) == 1
+    assert transfer[0].sources[0].rse.name in (tape1_rse_name, tape2_rse_name)
 
     # On equal source ranking, but different distance; the smaller distance is preferred
     [[_, [transfer]]] = pick_and_prepare_submission_path(topology=topology, protocol_factory=ProtocolFactory(),
                                                          requests_with_sources=requests).items()
-    assert len(transfer[0].legacy_sources) == 1
-    assert transfer[0].legacy_sources[0][0] == tape2_rse_name
+    assert len(transfer[0].sources) == 1
+    assert transfer[0].sources[0].rse.name == tape2_rse_name
 
     # On different source ranking, the bigger ranking is preferred
     tape2_source.ranking = -1
     [[_, [transfer]]] = pick_and_prepare_submission_path(topology=topology, protocol_factory=ProtocolFactory(),
                                                          requests_with_sources=requests).items()
-    assert len(transfer[0].legacy_sources) == 1
-    assert transfer[0].legacy_sources[0][0] == tape1_rse_name
+    assert len(transfer[0].sources) == 1
+    assert transfer[0].sources[0].rse.name == tape1_rse_name
 
 
 @pytest.mark.parametrize("file_config_mock", [