rucio-clients 34.4.0__tar.gz → 34.4.2__tar.gz

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 34.4.0
+Version: 34.4.2
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.9, <4
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: requests<=2.31.0,>=2.25.1
+Requires-Dist: requests~=2.32.0
 Requires-Dist: urllib3~=1.26.18
 Requires-Dist: dogpile.cache~=1.2.2
 Requires-Dist: tabulate~=0.9.0

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/client/accountclient.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from json import dumps
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import quote_plus
 
 from requests.status_codes import codes
@@ -20,6 +21,9 @@ from requests.status_codes import codes
 from rucio.client.baseclient import BaseClient, choice
 from rucio.common.utils import build_url
 
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
 
 class AccountClient(BaseClient):
 
@@ -27,7 +31,7 @@ class AccountClient(BaseClient):
 
     ACCOUNTS_BASEURL = 'accounts'
 
-    def add_account(self, account, type_, email):
+    def add_account(self, account: str, type_: str, email: str) -> bool:
         """
         Sends the request to create a new account.
 
@@ -49,7 +53,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def delete_account(self, account):
+    def delete_account(self, account: str) -> bool:
         """
         Sends the request to disable an account.
 
@@ -68,7 +72,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def get_account(self, account):
+    def get_account(self, account: str) -> Optional[dict[str, Any]]:
         """
         Sends the request to get information about a given account.
 
@@ -87,7 +91,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def update_account(self, account, key, value):
+    def update_account(self, account: str, key: str, value: Any) -> bool:
         """ Update a property of an account.
 
         :param account: Name of the account.
@@ -106,7 +110,12 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def list_accounts(self, account_type=None, identity=None, filters=None):
+    def list_accounts(
+            self,
+            account_type: Optional[str] = None,
+            identity: Optional[str] = None,
+            filters: Optional[dict[str, Any]] = None
+    ) -> "Iterator[dict[str, Any]]":
         """
         Sends the request to list all rucio accounts.
 
@@ -137,7 +146,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def whoami(self):
+    def whoami(self) -> Optional[dict[str, Any]]:
         """
         Get information about account whose token is used
 
@@ -146,7 +155,15 @@ class AccountClient(BaseClient):
         """
         return self.get_account('whoami')
 
-    def add_identity(self, account, identity, authtype, email, default=False, password=None):
+    def add_identity(
+            self,
+            account: str,
+            identity: str,
+            authtype: str,
+            email: str,
+            default: bool = False,
+            password: Optional[str] = None
+    ) -> bool:
         """
         Adds a membership association between identity and account.
 
@@ -171,7 +188,12 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def del_identity(self, account, identity, authtype):
+    def del_identity(
+            self,
+            account: str,
+            identity: str,
+            authtype: str
+    ) -> bool:
         """
         Delete an identity's membership association with an account.
 
@@ -194,7 +216,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def list_identities(self, account):
+    def list_identities(self, account: str) -> "Iterator[dict[str, Any]]":
         """
         List all identities on an account.
 
@@ -210,7 +232,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def list_account_rules(self, account):
+    def list_account_rules(self, account: str) -> "Iterator[dict[str, Any]]":
         """
         List the associated rules of an account.
 
@@ -226,7 +248,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def get_account_limits(self, account, rse_expression, locality):
+    def get_account_limits(self, account: str, rse_expression: str, locality: str) -> dict[str, Any]:
         """
         Return the correct account limits for the given locality.
 
@@ -243,7 +265,7 @@ class AccountClient(BaseClient):
             from rucio.common.exception import UnsupportedOperation
             raise UnsupportedOperation('The provided locality (%s) for the account limit was invalid' % locality)
 
-    def get_global_account_limit(self, account, rse_expression):
+    def get_global_account_limit(self, account: str, rse_expression: str) -> dict[str, Any]:
         """
         List the account limit for the specific RSE expression.
 
@@ -259,7 +281,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def get_global_account_limits(self, account):
+    def get_global_account_limits(self, account: str) -> dict[str, Any]:
         """
         List all RSE expression limits of this account.
 
@@ -274,7 +296,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def get_local_account_limits(self, account):
+    def get_local_account_limits(self, account: str) -> dict[str, Any]:
         """
         List the account rse limits of this account.
 
@@ -289,7 +311,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def get_local_account_limit(self, account, rse):
+    def get_local_account_limit(self, account: str, rse: str) -> dict[str, Any]:
         """
         List the account rse limits of this account for the specific rse.
 
@@ -305,7 +327,7 @@ class AccountClient(BaseClient):
         exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
         raise exc_cls(exc_msg)
 
-    def get_local_account_usage(self, account, rse=None):
+    def get_local_account_usage(self, account: str, rse: Optional[str] = None) -> "Iterator[dict[str, Any]]":
         """
         List the account usage for one or all rses of this account.
 
@@ -324,7 +346,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def get_global_account_usage(self, account, rse_expression=None):
+    def get_global_account_usage(self, account: str, rse_expression: Optional[str] = None) -> "Iterator[dict[str, Any]]":
         """
         List the account usage for one or all RSE expressions of this account.
 
@@ -343,7 +365,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def get_account_usage_history(self, account, rse):
+    def get_account_usage_history(self, account: str, rse: str) -> dict[str, Any]:
         """
         List the account usage history of this account on rse.
 
@@ -359,7 +381,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def list_account_attributes(self, account):
+    def list_account_attributes(self, account: str) -> "Iterator[dict[dict[str, Any], Any]]":
         """
         List the attributes for an account.
 
@@ -374,7 +396,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def add_account_attribute(self, account, key, value):
+    def add_account_attribute(self, account: str, key: str, value: Any) -> bool:
         """
         Adds an attribute to an account.
 
@@ -393,7 +415,7 @@ class AccountClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=res.headers, status_code=res.status_code, data=res.content)
             raise exc_cls(exc_msg)
 
-    def delete_account_attribute(self, account, key):
+    def delete_account_attribute(self, account: str, key: str) -> bool:
         """
         Delete an attribute for an account.
 

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/client/baseclient.py

@@ -879,8 +879,8 @@ class BaseClient:
             return False
 
         try:
-            token_file_handler = open(self.token_file, 'r')
-            self.auth_token = token_file_handler.readline()
+            with open(self.token_file, 'r') as token_file_handler:
+                self.auth_token = token_file_handler.readline()
             self.headers['X-Rucio-Auth-Token'] = self.auth_token
         except OSError as error:
             print("I/O error({0}): {1}".format(error.errno, error.strerror))

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/client/scopeclient.py

@@ -27,7 +27,11 @@ class ScopeClient(BaseClient):
 
     SCOPE_BASEURL = 'accounts'
 
-    def add_scope(self, account, scope):
+    def add_scope(
+            self,
+            account: str,
+            scope: str
+    ) -> bool:
         """
         Sends the request to add a new scope.
 
@@ -47,7 +51,7 @@ class ScopeClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_scopes(self):
+    def list_scopes(self) -> list[str]:
         """
         Sends the request to list all scopes.
 
@@ -64,7 +68,7 @@ class ScopeClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def list_scopes_for_account(self, account):
+    def list_scopes_for_account(self, account: str) -> list[str]:
         """
         Sends the request to list all scopes for a rucio account.
 

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/client/touchclient.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from json import dumps
+from typing import Optional
 
 from requests import post
 
@@ -30,7 +31,12 @@ class TouchClient(BaseClient):
     DIDS_BASEURL = 'dids'
     TRACES_BASEURL = 'traces'
 
-    def touch(self, scope, name, rse=None):
+    def touch(
+            self,
+            scope: str,
+            name: str,
+            rse: Optional[str] = None
+    ) -> None:
         """
         Sends a touch trace for a given file or dataset.
 

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/client/uploadclient.py

@@ -286,7 +286,6 @@ class UploadClient:
                     state_reason = str(error)
 
             if success:
-                num_succeeded += 1
                 trace['transferEnd'] = time.time()
                 trace['clientState'] = 'DONE'
                 file['state'] = 'A'
@@ -296,6 +295,7 @@ class UploadClient:
                 if summary_file_path:
                     summary.append(copy.deepcopy(file))
 
+                registration_succeeded = True
                 if not no_register:
                     if register_after_upload:
                         self._register_file(file, registered_dataset_dids, ignore_availability=ignore_availability, activity=activity)
@@ -304,6 +304,7 @@ class UploadClient:
                         try:
                             self.client.update_replicas_states(rse, files=[replica_for_api])
                         except Exception as error:
+                            registration_succeeded = False
                             logger(logging.ERROR, 'Failed to update replica state for file {}'.format(basename))
                             logger(logging.DEBUG, 'Details: {}'.format(str(error)))
 
@@ -312,8 +313,13 @@ class UploadClient:
                     try:
                         self.client.attach_dids(file['dataset_scope'], file['dataset_name'], [file_did])
                     except Exception as error:
-                        logger(logging.WARNING, 'Failed to attach file to the dataset')
+                        registration_succeeded = False
+                        logger(logging.ERROR, 'Failed to attach file to the dataset')
                         logger(logging.DEBUG, 'Attaching to dataset {}'.format(str(error)))
+
+                # only report success if the registration operations succeeded as well
+                if registration_succeeded:
+                    num_succeeded += 1
             else:
                 trace['clientState'] = 'FAILED'
                 trace['stateReason'] = state_reason

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/common/types.py

@@ -24,7 +24,7 @@ class InternalType:
     '''
     Base for Internal representations of string types
     '''
-    def __init__(self, value, vo='def', fromExternal=True):
+    def __init__(self, value: Optional[str], vo: str = 'def', fromExternal: bool = True):
         if value is None:
             self.external = None
             self.internal = None
@@ -73,20 +73,22 @@ class InternalType:
     def __hash__(self):
         return hash(self.internal)
 
-    def _calc_external(self):
+    def _calc_external(self) -> tuple[str, str]:
         ''' Utility to convert between internal and external representations'''
-        split = self.internal.split('@', 1)
-        if len(split) == 1:  # if cannot convert, vo is '' and this is single vo
-            vo = 'def'
-            external = split[0]
-        else:
-            vo = split[1]
-            external = split[0]
-        return vo, external
-
-    def _calc_internal(self):
+        if isinstance(self.internal, str):
+            split = self.internal.split('@', 1)
+            if len(split) == 1:  # if cannot convert, vo is '' and this is single vo
+                vo = 'def'
+                external = split[0]
+            else:
+                vo = split[1]
+                external = split[0]
+            return vo, external
+        return '', ''
+
+    def _calc_internal(self) -> str:
         ''' Utility to convert between internal and external representations'''
-        if self.vo == 'def':
+        if self.vo == 'def' and self.external is not None:
             return self.external
         internal = '{}@{}'.format(self.external, self.vo)
         return internal
@@ -96,7 +98,7 @@ class InternalAccount(InternalType):
     '''
     Internal representation of an account
     '''
-    def __init__(self, account, vo='def', fromExternal=True):
+    def __init__(self, account: Optional[str], vo: str = 'def', fromExternal: bool = True):
         super(InternalAccount, self).__init__(value=account, vo=vo, fromExternal=fromExternal)
 
 
@@ -104,7 +106,7 @@ class InternalScope(InternalType):
     '''
     Internal representation of a scope
     '''
-    def __init__(self, scope, vo='def', fromExternal=True):
+    def __init__(self, scope: Optional[str], vo: str = 'def', fromExternal: bool = True):
         super(InternalScope, self).__init__(value=scope, vo=vo, fromExternal=fromExternal)
 
 

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/rse/protocols/protocol.py

@@ -16,9 +16,9 @@
 This module defines the base class for implementing a transfer protocol,
 along with some of the default methods for LFN2PFN translations.
 """
-
 import hashlib
 import logging
+from collections.abc import Callable, Mapping
 from configparser import NoOptionError, NoSectionError
 from typing import TypeVar
 from urllib.parse import urlparse
@@ -37,6 +37,61 @@ if getattr(rsemanager, 'SERVER_MODE', None):
     from rucio.core.rse import get_rse_vo
 
 
+class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
+    """
+        Translates a pfn dictionary into a scope and name
+    """
+    def __init__(self, vo: str = 'def'):
+        super().__init__()
+        self.register("def", RSEDeterministicScopeTranslation._default)
+        self.register("atlas", RSEDeterministicScopeTranslation._atlas)
+        policy_module = vo
+        # Uses the same policy as the DeterministicTranslation
+        if super()._supports(self.__class__.__name__, policy_module):
+            self.parser = self._get_one_algorithm(self.__class__.__name__, policy_module)
+        else:
+            self.parser = self._get_one_algorithm(self.__class__.__name__, "def")
+
+    @classmethod
+    def register(cls, name: str, func: Callable) -> None:
+        super()._register(cls.__name__, {name: func})
+
+    @staticmethod
+    def _default(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
+        """ Translate pfn to name/scope pair
+
+        :param parsed_pfn: dictionary representing pfn containing:
+            - path: str,
+            - name: str
+        :return: tuple containing name, scope
+        """
+        path = parsed_pfn['path']
+        scope = path.lstrip('/').split('/')[0]
+        name = parsed_pfn['name']
+        return name, scope
+
+    @staticmethod
+    def _atlas(parsed_pfn: Mapping[str, str]) -> tuple[str, str]:
+        """ Translate pfn to name/scope pair
+
+        :param parsed_pfn: dictionary representing pfn containing:
+            - path: str,
+            - name: str
+        :return: tuple containing name, scope
+        """
+        path = parsed_pfn['path']
+        if path.startswith('/user') or path.startswith('/group'):
+            scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
+            name = parsed_pfn['name']
+        else:
+            name, scope = RSEDeterministicScopeTranslation._default(parsed_pfn)
+
+        return name, scope
+
+
+RSEDeterministicScopeTranslation()
+
+
 RSEDeterministicTranslationT = TypeVar('RSEDeterministicTranslationT', bound='RSEDeterministicTranslation')
 
 

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/rse/rsemanager.py

@@ -15,6 +15,7 @@
 import copy
 import logging
 import random
+from collections.abc import Callable
 from time import sleep
 from urllib.parse import urlparse
 
@@ -25,6 +26,18 @@ from rucio.common.logging import formatted_logger
 from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, make_valid_did
 
 
+def get_scope_protocol(vo: str = 'def') -> Callable:
+    """
+        Returns the callable protocol to translate the pfn to a name/scope pair
+
+    :returns:
+        Callable: Scope Parser function
+    """
+    from rucio.rse.protocols.protocol import RSEDeterministicScopeTranslation
+    translation = RSEDeterministicScopeTranslation(vo=vo)
+    return translation.parser
+
+
 def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict:
     """
         Returns all protocol related RSE attributes.

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '34.4.0',
+    'version': '34.4.2',
     'branch_nick': 'release-34',
-    'revision_id': 'e9aff7ed5dc1ba4c1085faa212d61a875d1adbd1',
-    'revno': 12859
+    'revision_id': '534cf95bc494f948d820be9410ac498e1dab1cbd',
+    'revno': 12877
 }

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/pyproject.toml

@@ -56,6 +56,7 @@ ignore = [
     "UP030", # Use implicit references for positional format fields
     "UP031", # Use format specifiers instead of percent format
     "UP032", # Use f-string instead of `format` call
+    "UP037", # Remove quotes from type annotation. Seeing possible false positives caused by https://github.com/astral-sh/ruff/pull/11485
     "S101",  # Pending https://github.com/rucio/rucio/issues/6680
     "S105",  # Pending https://github.com/rucio/rucio/issues/6696
     "S108",  # Pending https://github.com/rucio/rucio/issues/6655

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/requirements.txt

@@ -1,5 +1,5 @@
 # All dependencies needed to run rucio client (and server/daemons) should be defined here
-requests>=2.25.1,<=2.31.0                                   # Python HTTP for Humans.
+requests~=2.32.0                                            # Python HTTP for Humans.
 urllib3~=1.26.18                                            # HTTP library with thread-safe connection pooling, file post, etc.
 dogpile.cache~=1.2.2                                        # Caching API plugins (1.1.2 is the first version to support pymemcache)
 tabulate~=0.9.0                                             # Pretty-print tabular data

{rucio_clients-34.4.0 → rucio_clients-34.4.2}/tests/test_upload.py

@@ -21,10 +21,11 @@ from unittest.mock import patch
 
 import pytest
 
+from rucio.client.client import Client
 from rucio.client.uploadclient import UploadClient
 from rucio.common.config import config_add_section, config_set
 from rucio.common.constants import RseAttr
-from rucio.common.exception import InputValidationError, NoFilesUploaded, NotAllFilesUploaded
+from rucio.common.exception import InputValidationError, NoFilesUploaded, NotAllFilesUploaded, ResourceTemporaryUnavailable
 from rucio.common.utils import adler32, generate_uuid
 from rucio.core.rse import add_protocol, add_rse_attribute
 
@@ -398,3 +399,33 @@ def test_upload_file_ignore_availability(rse_factory, scope, upload_client, file
 
     status = upload_client.upload(item, ignore_availability=True)
     assert status == 0
+
+
+@pytest.fixture
+def upload_client_registration_fail():
+    logger = logging.getLogger('upload_client')
+    logger.addHandler(logging.StreamHandler())
+    logger.setLevel(logging.DEBUG)
+    # modify the client object used by upload_client so that replica registration fails
+    class RegistrationFailureClient(Client):
+        def __init__(self, **args):
+            super(RegistrationFailureClient, self).__init__(**args)
+        def update_replicas_states(self, rse, files):
+            # simulate server timing out
+            raise ResourceTemporaryUnavailable
+    return UploadClient(logger=logger, _client=RegistrationFailureClient())
+
+
+def test_upload_registration_fail(rse, scope, upload_client_registration_fail, file_factory):
+    local_file = file_factory.file_generator()
+    fn = os.path.basename(local_file)
+
+    # upload a file and check that exception is raised
+    with pytest.raises(NoFilesUploaded):
+        upload_client_registration_fail.upload([{
+            'path': local_file,
+            'rse': rse,
+            'did_scope': scope,
+            'did_name': fn,
+            'guid': generate_uuid()
+        }])