rucio 34.4.0__tar.gz → 34.4.1__tar.gz

{rucio-34.4.0 → rucio-34.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 34.4.0
+Version: 34.4.1
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio
@@ -20,7 +20,7 @@ Classifier: Environment :: No Input/Output (Daemon)
 Requires-Python: >=3.9, <4
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: requests<=2.31.0,>=2.25.1
+Requires-Dist: requests~=2.32.0
 Requires-Dist: urllib3~=1.26.18
 Requires-Dist: dogpile.cache~=1.2.2
 Requires-Dist: tabulate~=0.9.0

{rucio-34.4.0 → rucio-34.4.1}/lib/rucio/client/uploadclient.py

@@ -286,7 +286,6 @@ class UploadClient:
                     state_reason = str(error)
 
             if success:
-                num_succeeded += 1
                 trace['transferEnd'] = time.time()
                 trace['clientState'] = 'DONE'
                 file['state'] = 'A'
@@ -296,6 +295,7 @@ class UploadClient:
                 if summary_file_path:
                     summary.append(copy.deepcopy(file))
 
+                registration_succeeded = True
                 if not no_register:
                     if register_after_upload:
                         self._register_file(file, registered_dataset_dids, ignore_availability=ignore_availability, activity=activity)
@@ -304,6 +304,7 @@ class UploadClient:
                         try:
                             self.client.update_replicas_states(rse, files=[replica_for_api])
                         except Exception as error:
+                            registration_succeeded = False
                             logger(logging.ERROR, 'Failed to update replica state for file {}'.format(basename))
                             logger(logging.DEBUG, 'Details: {}'.format(str(error)))
 
@@ -312,8 +313,13 @@ class UploadClient:
                     try:
                         self.client.attach_dids(file['dataset_scope'], file['dataset_name'], [file_did])
                     except Exception as error:
-                        logger(logging.WARNING, 'Failed to attach file to the dataset')
+                        registration_succeeded = False
+                        logger(logging.ERROR, 'Failed to attach file to the dataset')
                         logger(logging.DEBUG, 'Attaching to dataset {}'.format(str(error)))
+
+                # only report success if the registration operations succeeded as well
+                if registration_succeeded:
+                    num_succeeded += 1
             else:
                 trace['clientState'] = 'FAILED'
                 trace['stateReason'] = state_reason

{rucio-34.4.0 → rucio-34.4.1}/lib/rucio/core/replica.py

@@ -268,20 +268,11 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
         rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
         proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
         if rse_info['deterministic']:
-            # TBD : In case of deterministic RSE, call the extract_scope_from_path method
+            scope_proto = rsemgr.get_scope_protocol(vo=issuer.vo)
             parsed_pfn = proto.parse_pfns(pfns=pfns)
             for pfn in parsed_pfn:
-                # WARNING : this part is ATLAS specific and must be changed
-                path = parsed_pfn[pfn]['path']
-                if path.startswith('/user') or path.startswith('/group'):
-                    scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
-                    name = parsed_pfn[pfn]['name']
-                elif path.startswith('/'):
-                    scope = path.split('/')[1]
-                    name = parsed_pfn[pfn]['name']
-                else:
-                    scope = path.split('/')[0]
-                    name = parsed_pfn[pfn]['name']
+                # Translate into a scope and name
+                name, scope = scope_proto(parsed_pfn[pfn])
 
                 scope = InternalScope(scope, vo=issuer.vo)
                 replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': status})
@@ -600,20 +591,12 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
         pfndict = {}
         proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
         if rse_info['deterministic']:
+            scope_proto = rsemgr.get_scope_protocol(vo=vo)
             parsed_pfn = proto.parse_pfns(pfns=pfns)
 
-            # WARNING : this part is ATLAS specific and must be changed
             for pfn in parsed_pfn:
-                path = parsed_pfn[pfn]['path']
-                if path.startswith('/user') or path.startswith('/group'):
-                    scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
-                    name = parsed_pfn[pfn]['name']
-                elif path.startswith('/'):
-                    scope = path.split('/')[1]
-                    name = parsed_pfn[pfn]['name']
-                else:
-                    scope = path.split('/')[0]
-                    name = parsed_pfn[pfn]['name']
+                # Translate into a scope and name
+                name, scope = scope_proto(parsed_pfn[pfn])
                 scope = InternalScope(scope, vo)
                 yield {pfn: {'scope': scope, 'name': name}}
         else:

{rucio-34.4.0 → rucio-34.4.1}/lib/rucio/rse/protocols/protocol.py

@@ -16,9 +16,9 @@
 This module defines the base class for implementing a transfer protocol,
 along with some of the default methods for LFN2PFN translations.
 """
-
 import hashlib
 import logging
+from collections.abc import Callable, Mapping
 from configparser import NoOptionError, NoSectionError
 from typing import TypeVar
 from urllib.parse import urlparse
@@ -37,6 +37,61 @@ if getattr(rsemanager, 'SERVER_MODE', None):
     from rucio.core.rse import get_rse_vo
 
 
+class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
+    """
+        Translates a pfn dictionary into a scope and name
+    """
+    def __init__(self, vo: str = 'def'):
+        super().__init__()
+        self.register("def", RSEDeterministicScopeTranslation._default)
+        self.register("atlas", RSEDeterministicScopeTranslation._atlas)
+        policy_module = vo
+        # Uses the same policy as the DeterministicTranslation
+        if super()._supports(self.__class__.__name__, policy_module):
+            self.parser = self._get_one_algorithm(self.__class__.__name__, policy_module)
+        else:
+            self.parser = self._get_one_algorithm(self.__class__.__name__, "def")
+
+    @classmethod
+    def register(cls, name: str, func: Callable) -> None:
+        super()._register(cls.__name__, {name: func})
+
+    @staticmethod
+    def _default(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
+        """ Translate pfn to name/scope pair
+
+        :param parsed_pfn: dictionary representing pfn containing:
+            - path: str,
+            - name: str
+        :return: tuple containing name, scope
+        """
+        path = parsed_pfn['path']
+        scope = path.lstrip('/').split('/')[0]
+        name = parsed_pfn['name']
+        return name, scope
+
+    @staticmethod
+    def _atlas(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
+        """ Translate pfn to name/scope pair
+
+        :param parsed_pfn: dictionary representing pfn containing:
+            - path: str,
+            - name: str
+        :return: tuple containing name, scope
+        """
+        path = parsed_pfn['path']
+        if path.startswith('/user') or path.startswith('/group'):
+            scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
+            name = parsed_pfn['name']
+        else:
+            name, scope = RSEDeterministicScopeTranslation._default(parsed_pfn)
+
+        return name, scope
+
+
+RSEDeterministicScopeTranslation()
+
+
 RSEDeterministicTranslationT = TypeVar('RSEDeterministicTranslationT', bound='RSEDeterministicTranslation')
 
 

{rucio-34.4.0 → rucio-34.4.1}/lib/rucio/rse/rsemanager.py

@@ -15,6 +15,7 @@
 import copy
 import logging
 import random
+from collections.abc import Callable
 from time import sleep
 from urllib.parse import urlparse
 
@@ -25,6 +26,18 @@ from rucio.common.logging import formatted_logger
 from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, make_valid_did
 
 
+def get_scope_protocol(vo: str = 'def') -> Callable:
+    """
+        Returns the callable protocol to translate the pfn to a name/scope pair
+
+    :returns:
+        Callable: Scope Parser function
+    """
+    from rucio.rse.protocols.protocol import RSEDeterministicScopeTranslation
+    translation = RSEDeterministicScopeTranslation(vo=vo)
+    return translation.parser
+
+
 def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict:
     """
         Returns all protocol related RSE attributes.

{rucio-34.4.0 → rucio-34.4.1}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '34.4.0',
+    'version': '34.4.1',
     'branch_nick': 'release-34',
-    'revision_id': 'e9aff7ed5dc1ba4c1085faa212d61a875d1adbd1',
-    'revno': 12859
+    'revision_id': '6699a667cf3cd9785086857295f70bc66c7c2174',
+    'revno': 12863
 }

{rucio-34.4.0 → rucio-34.4.1}/requirements.txt

@@ -1,5 +1,5 @@
 # All dependencies needed to run rucio client (and server/daemons) should be defined here
-requests>=2.25.1,<=2.31.0                                   # Python HTTP for Humans.
+requests~=2.32.0                                            # Python HTTP for Humans.
 urllib3~=1.26.18                                            # HTTP library with thread-safe connection pooling, file post, etc.
 dogpile.cache~=1.2.2                                        # Caching API plugins (1.1.2 is the first version to support pymemcache)
 tabulate~=0.9.0                                             # Pretty-print tabular data

{rucio-34.4.0 → rucio-34.4.1}/tests/test_upload.py

@@ -21,10 +21,11 @@ from unittest.mock import patch
 
 import pytest
 
+from rucio.client.client import Client
 from rucio.client.uploadclient import UploadClient
 from rucio.common.config import config_add_section, config_set
 from rucio.common.constants import RseAttr
-from rucio.common.exception import InputValidationError, NoFilesUploaded, NotAllFilesUploaded
+from rucio.common.exception import InputValidationError, NoFilesUploaded, NotAllFilesUploaded, ResourceTemporaryUnavailable
 from rucio.common.utils import adler32, generate_uuid
 from rucio.core.rse import add_protocol, add_rse_attribute
 
@@ -398,3 +399,33 @@ def test_upload_file_ignore_availability(rse_factory, scope, upload_client, file
 
     status = upload_client.upload(item, ignore_availability=True)
     assert status == 0
+
+
+@pytest.fixture
+def upload_client_registration_fail():
+    logger = logging.getLogger('upload_client')
+    logger.addHandler(logging.StreamHandler())
+    logger.setLevel(logging.DEBUG)
+    # modify the client object used by upload_client so that replica registration fails
+    class RegistrationFailureClient(Client):
+        def __init__(self, **args):
+            super(RegistrationFailureClient, self).__init__(**args)
+        def update_replicas_states(self, rse, files):
+            # simulate server timing out
+            raise ResourceTemporaryUnavailable
+    return UploadClient(logger=logger, _client=RegistrationFailureClient())
+
+
+def test_upload_registration_fail(rse, scope, upload_client_registration_fail, file_factory):
+    local_file = file_factory.file_generator()
+    fn = os.path.basename(local_file)
+
+    # upload a file and check that exception is raised
+    with pytest.raises(NoFilesUploaded):
+        upload_client_registration_fail.upload([{
+            'path': local_file,
+            'rse': rse,
+            'did_scope': scope,
+            'did_name': fn,
+            'guid': generate_uuid()
+        }])