rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/rse/protocols/webdav.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,11 +15,11 @@
 import os
 import sys
 import xml.etree.ElementTree as ET
-from typing import Optional
+from dataclasses import dataclass
+from typing import Any, Optional
 from urllib.parse import urlparse
 
 import requests
-from dataclasses import dataclass
 from requests.adapters import HTTPAdapter
 from urllib3.poolmanager import PoolManager
 
@@ -40,7 +39,7 @@ class TLSHTTPAdapter(HTTPAdapter):
                                        ca_cert_dir="/etc/grid-security/certificates")
 
 
-class UploadInChunks(object):
+class UploadInChunks:
     '''
     Class to upload by chunks.
     '''
@@ -73,7 +72,7 @@ class UploadInChunks(object):
         return self.__totalsize
 
 
-class IterableToFileAdapter(object):
+class IterableToFileAdapter:
     '''
     Class IterableToFileAdapter
     '''
@@ -134,7 +133,7 @@ class _PropfindResponse:
         """
 
         try:
-            xml = ET.fromstring(document)
+            xml = ET.fromstring(document)  # noqa: S314
         except ET.ParseError as ex:
             raise ValueError("Couldn't parse XML document") from ex
 
@@ -152,7 +151,7 @@ class Default(protocol.RSEProtocol):
 
     """ Implementing access to RSEs using the webDAV protocol."""
 
-    def connect(self, credentials={}):
+    def connect(self, credentials: Optional[dict[str, Any]] = None) -> None:
         """ Establishes the actual connection to the referred RSE.
 
             :param credentials: Provides information to establish a connection
@@ -161,6 +160,7 @@ class Default(protocol.RSEProtocol):
 
             :raises RSEAccessDenied
         """
+        credentials = credentials or {}
         try:
             parse_url = urlparse(self.path2pfn(''))
             self.server = f'{parse_url.scheme}://{parse_url.netloc}'
@@ -185,12 +185,15 @@ class Default(protocol.RSEProtocol):
                 # Trying to get the proxy from the default location
                 proxy_path = '/tmp/x509up_u%s' % os.geteuid()
                 if os.path.isfile(proxy_path):
-                    x509 = proxy_path
+                    self.cert = (proxy_path, proxy_path)
                 elif self.auth_token:
+                    # If no proxy is found, we set the cert to None and use the auth_token
+                    self.cert = None
                     pass
                 else:
                     raise exception.RSEAccessDenied('X509_USER_PROXY is not set')
-            self.cert = (x509, x509)
+            else:
+                self.cert = (x509, x509)
 
         try:
             self.timeout = credentials['timeout']
@@ -332,13 +335,13 @@ class Default(protocol.RSEProtocol):
                         raise exception.RucioException(result.status_code, result.text)
                 except requests.exceptions.ConnectionError as error:
                     raise exception.ServiceUnavailable(error)
-                except IOError as error:
+                except OSError as error:
                     raise exception.SourceNotFound(error)
         except requests.exceptions.ConnectionError as error:
             raise exception.ServiceUnavailable(error)
         except requests.exceptions.ReadTimeout as error:
             raise exception.ServiceUnavailable(error)
-        except IOError as error:
+        except OSError as error:
             raise exception.SourceNotFound(error)
 
     def rename(self, pfn, new_pfn):
@@ -479,7 +482,7 @@ class Default(protocol.RSEProtocol):
 
             :param path: path to file
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
             :raises RSEAccessDenied: in case of permission issue.
 
@@ -528,13 +531,13 @@ class Default(protocol.RSEProtocol):
 
         :returns: a list with dict containing 'totalsize' and 'unusedsize'
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         """
         endpoint_basepath = self.path2pfn('')
         headers = {'Depth': '0'}
 
         try:
-            root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text)
+            root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text)  # noqa: S314
             usedsize = root[0][1][0].find('{DAV:}quota-used-bytes').text
             try:
                 unusedsize = root[0][1][0].find('{DAV:}quota-available-bytes').text

rucio/rse/protocols/xrootd.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,13 +16,20 @@ import logging
 import os
 
 from rucio.common import exception
-from rucio.common.utils import execute, PREFERRED_CHECKSUM
+from rucio.common.utils import PREFERRED_CHECKSUM, execute
 from rucio.rse.protocols import protocol
 
 
 class Default(protocol.RSEProtocol):
     """ Implementing access to RSEs using the XRootD protocol using GSI authentication."""
 
+    @property
+    def _auth_env(self):
+        if self.auth_token:
+            return f"XrdSecPROTOCOL=ztn BEARER_TOKEN='{self.auth_token}'"
+        else:
+            return 'XrdSecPROTOCOL=gsi'
+
     def __init__(self, protocol_attr, rse_settings, logger=logging.log):
         """ Initializes the object with information about the referred RSE.
 
@@ -66,7 +72,7 @@ class Default(protocol.RSEProtocol):
         self.logger(logging.DEBUG, 'xrootd.exists: pfn: {}'.format(pfn))
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.exists: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -82,11 +88,11 @@ class Default(protocol.RSEProtocol):
 
         :param path: path to file
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
 
         :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
-        self.logger(logging.DEBUG, 'xrootd.stat: path: {}'.format(path))
+        self.logger(logging.DEBUG, f'xrootd.stat: path: {path}')
         ret = {}
         chsum = None
         if path.startswith('root:'):
@@ -94,7 +100,7 @@ class Default(protocol.RSEProtocol):
 
         try:
             # xrdfs stat for getting filesize
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: filesize cmd: {}'.format(cmd))
             status_stat, out, err = execute(cmd)
             if status_stat == 0:
@@ -106,7 +112,7 @@ class Default(protocol.RSEProtocol):
                             break
 
             # xrdfs query checksum for getting checksum
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s query checksum %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} query checksum {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: checksum cmd: {}'.format(cmd))
             status_query, out, err = execute(cmd)
             if status_query == 0:
@@ -160,7 +166,7 @@ class Default(protocol.RSEProtocol):
         if not prefix.endswith('/'):
             prefix = ''.join([prefix, '/'])
 
-        lfns = [lfns] if type(lfns) == dict else lfns
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
         for lfn in lfns:
             scope, name = lfn['scope'], lfn['name']
             if 'path' in lfn and lfn['path'] is not None:
@@ -183,7 +189,7 @@ class Default(protocol.RSEProtocol):
         try:
             # The query stats call is not implemented on some xroot doors.
             # Workaround: fail, if server does not reply within 10 seconds for static config query
-            cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
+            cmd = f'{self._auth_env} XRD_REQUESTTIMEOUT=10 xrdfs {self.hostname}:{self.port} query config {self.hostname}:{self.port}'
             self.logger(logging.DEBUG, 'xrootd.connect: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -206,7 +212,7 @@ class Default(protocol.RSEProtocol):
         """
         self.logger(logging.DEBUG, 'xrootd.get: pfn: {}'.format(pfn))
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
+            cmd = f'{self._auth_env} xrdcp -f {pfn} {dest}'
             self.logger(logging.DEBUG, 'xrootd.get: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status == 54:
@@ -226,7 +232,7 @@ class Default(protocol.RSEProtocol):
             :param transfer_timeout: Transfer timeout (in seconds) - dummy
 
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.put: filename: {} target: {}'.format(filename, target))
@@ -237,7 +243,7 @@ class Default(protocol.RSEProtocol):
         if not os.path.exists(source_url):
             raise exception.SourceNotFound()
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
+            cmd = f'{self._auth_env} xrdcp -f {source_url} {path}'
             self.logger(logging.DEBUG, 'xrootd.put: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -251,7 +257,7 @@ class Default(protocol.RSEProtocol):
 
             :param pfn: Physical file name
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.delete: pfn: {}'.format(pfn))
@@ -259,7 +265,7 @@ class Default(protocol.RSEProtocol):
             raise exception.SourceNotFound()
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} rm {path}'
             self.logger(logging.DEBUG, 'xrootd.delete: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -273,7 +279,7 @@ class Default(protocol.RSEProtocol):
             :param pfn:      Current physical file name
             :param new_pfn  New physical file name
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.rename: pfn: {}'.format(pfn))
@@ -283,10 +289,10 @@ class Default(protocol.RSEProtocol):
             path = self.pfn2path(pfn)
             new_path = self.pfn2path(new_pfn)
             new_dir = new_path[:new_path.rindex('/') + 1]
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mkdir -p {new_dir}'
             self.logger(logging.DEBUG, 'xrootd.stat: mkdir cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mv {path} {new_path}'
             self.logger(logging.DEBUG, 'xrootd.stat: rename cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:

rucio/rse/rsemanager.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,15 +15,28 @@
 import copy
 import logging
 import random
+from collections.abc import Callable
 from time import sleep
 from urllib.parse import urlparse
 
-from rucio.common import exception, utils, constants
-from rucio.common import types
+from rucio.common import constants, exception, types, utils
 from rucio.common.config import config_get_int
+from rucio.common.constants import RSE_SUPPORTED_PROTOCOL_OPERATIONS
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.logging import formatted_logger
-from rucio.common.utils import make_valid_did, GLOBALLY_SUPPORTED_CHECKSUMS
+from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, make_valid_did
+
+
+def get_scope_protocol(vo: str = 'def') -> Callable:
+    """
+        Returns the callable protocol to translate the pfn to a name/scope pair
+
+    :returns:
+        Callable: Scope Parser function
+    """
+    from rucio.rse.protocols.protocol import RSEDeterministicScopeTranslation
+    translation = RSEDeterministicScopeTranslation(vo=vo)
+    return translation.parser
 
 
 def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict:
@@ -118,7 +130,7 @@ def _get_possible_protocols(rse_settings: types.RSESettingsDict, operation, sche
 
 
 def get_protocols_ordered(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan', impl=None):
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():
@@ -130,7 +142,7 @@ def get_protocols_ordered(rse_settings: types.RSESettingsDict, operation, scheme
 
 
 def select_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan'):
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():
@@ -158,7 +170,7 @@ def create_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None,
 
     # Verify feasibility of Protocol
     operation = operation.lower()
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():

rucio/tests/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

rucio/tests/common.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,15 +18,19 @@ import json
 import os
 import tempfile
 from collections import namedtuple
+from collections.abc import Callable, Iterable
+from functools import wraps
+from os import rename
 from random import choice, choices
-from string import ascii_uppercase, ascii_letters, digits
-from typing import Optional
+from string import ascii_letters, ascii_uppercase, digits
+from typing import Any, Optional
 
 import pytest
 import requests
 
 from rucio.common.config import config_get, config_get_bool, get_config_dirs
-from rucio.common.utils import generate_uuid as uuid, execute
+from rucio.common.utils import execute
+from rucio.common.utils import generate_uuid as uuid
 
 skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
                                                   reason='fails if no rse-accounts.cfg found')
@@ -126,27 +129,27 @@ def rse_name_generator(size: int = 10) -> str:
 
     :returns: A random RSE name
     """
-    return 'MOCK-' + ''.join(choice(ascii_uppercase) for x in range(size))
+    return 'MOCK-' + ''.join(choice(ascii_uppercase) for x in range(size))  # noqa: S311
 
 
-def rfc2253_dn_generator():
+def rfc2253_dn_generator() -> str:
     """ Generate a random DN in RFC 2253 format.
 
     :returns: A random DN
     """
-    random_cn = ''.join(choices(ascii_letters + digits, k=8))
-    random_o = ''.join(choices(ascii_letters + digits, k=8))
-    random_c = ''.join(choices(ascii_letters, k=2))
+    random_cn = ''.join(choices(ascii_letters + digits, k=8))  # noqa: S311
+    random_o = ''.join(choices(ascii_letters + digits, k=8))  # noqa: S311
+    random_c = ''.join(choices(ascii_letters, k=2))  # noqa: S311
     random_dn = "CN={}, O={}, C={}".format(random_cn, random_o, random_c)
     return random_dn
 
 
-def file_generator(size: int = 2, namelen: int = 10):
+def file_generator(size: int = 2, namelen: int = 10) -> str:
     """ Create a bogus file and returns it's name.
     :param size: size in bytes
     :returns: The name of the generated file.
     """
-    fn = '/tmp/file_' + ''.join(choice(ascii_uppercase) for x in range(namelen))
+    fn = '/tmp/file_' + ''.join(choice(ascii_uppercase) for x in range(namelen))  # noqa: S311
     execute('dd if=/dev/urandom of={0} count={1} bs=1'.format(fn, size))
     return fn
 
@@ -154,7 +157,7 @@ def file_generator(size: int = 2, namelen: int = 10):
 def make_temp_file(dir_: str, data: str) -> str:
     """
     Creates a temporal file and write `data` on it.
-    :param data: String to be writen on the created file.
+    :param data: String to be written on the created file.
     :returns: Name of the temporal file.
     """
     fd, path = tempfile.mkstemp(dir=dir_)
@@ -199,11 +202,11 @@ def print_response(rest_response):
     print(text if text else '<no content>')
 
 
-def headers(*iterables):
+def headers(*iterables: Iterable):
     return list(itertools.chain(*iterables))
 
 
-def loginhdr(account, username, password):
+def loginhdr(account: str, username: str, password: str):
     yield 'X-Rucio-Account', str(account)
     yield 'X-Rucio-Username', str(username)
     yield 'X-Rucio-Password', str(password)
@@ -213,12 +216,12 @@ def auth(token):
     yield 'X-Rucio-Auth-Token', str(token)
 
 
-def vohdr(vo):
+def vohdr(vo: str):
     if vo:
         yield 'X-Rucio-VO', str(vo)
 
 
-def hdrdict(dictionary):
+def hdrdict(dictionary: dict):
     for key in dictionary:
         yield str(key), str(dictionary[key])
 
@@ -235,10 +238,33 @@ class Mime:
     BINARY = 'application/octet-stream'
 
 
-def load_test_conf_file(file_name):
+def load_test_conf_file(file_name: str) -> dict[str, Any]:
     config_dir = next(filter(lambda d: os.path.exists(os.path.join(d, file_name)), get_config_dirs()))
     with open(os.path.join(config_dir, file_name)) as f:
         return json.load(f)
 
 
+def remove_config(func: Callable) -> Callable:
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        for configfile in get_config_dirs():
+            # Rename the config to <config>.tmp
+            try:
+                rename(f"{configfile}rucio.cfg", f"{configfile}rucio.cfg.tmp")
+            except FileNotFoundError:
+                pass  # When a test uses a os.env assigned conf, there's nothing stating the default location has something
+        try:
+            # Execute the test
+            func(*args, **kwargs)
+        finally:
+            # And put the config back
+            for configfile in get_config_dirs():
+                try:
+                    rename(f"{configfile}rucio.cfg.tmp", f"{configfile}rucio.cfg")
+                except FileNotFoundError:
+                    pass
+
+    return wrapper
+
+
 RSE_namedtuple = namedtuple('RSE_namedtuple', ['name', 'id'])

rucio/tests/common_server.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,7 +19,8 @@ from sqlalchemy.orm import aliased
 from rucio.core import config as core_config
 from rucio.core.vo import map_vo
 from rucio.db.sqla import models
-from rucio.db.sqla.session import transactional_session, get_session
+from rucio.db.sqla.session import get_session, transactional_session
+
 from .common import get_long_vo
 
 # Functions containing server-only includes that can't be included in client tests
@@ -115,7 +115,7 @@ def cleanup_db_deps(model, select_rows_stmt, *, session=None):
 
 
 def reset_config_table():
-    """ Clear the config table and install any default entires needed for the tests.
+    """ Clear the config table and install any default entries needed for the tests.
     """
     db_session = get_session()
     db_session.query(models.Config).delete()

rucio/transfertool/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");