rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/rse/protocols/webdav.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,11 +15,11 @@
 import os
 import sys
 import xml.etree.ElementTree as ET
-from typing import Optional
+from dataclasses import dataclass
+from typing import Any, Optional
 from urllib.parse import urlparse
 
 import requests
-from dataclasses import dataclass
 from requests.adapters import HTTPAdapter
 from urllib3.poolmanager import PoolManager
 
@@ -40,7 +39,7 @@ class TLSHTTPAdapter(HTTPAdapter):
                                        ca_cert_dir="/etc/grid-security/certificates")
 
 
-class UploadInChunks(object):
+class UploadInChunks:
     '''
     Class to upload by chunks.
     '''
@@ -73,7 +72,7 @@ class UploadInChunks(object):
         return self.__totalsize
 
 
-class IterableToFileAdapter(object):
+class IterableToFileAdapter:
     '''
     Class IterableToFileAdapter
     '''
@@ -134,7 +133,7 @@ class _PropfindResponse:
         """
 
         try:
-            xml = ET.fromstring(document)
+            xml = ET.fromstring(document)  # noqa: S314
         except ET.ParseError as ex:
             raise ValueError("Couldn't parse XML document") from ex
 
@@ -152,7 +151,7 @@ class Default(protocol.RSEProtocol):
 
     """ Implementing access to RSEs using the webDAV protocol."""
 
-    def connect(self, credentials={}):
+    def connect(self, credentials: Optional[dict[str, Any]] = None) -> None:
         """ Establishes the actual connection to the referred RSE.
 
             :param credentials: Provides information to establish a connection
@@ -161,6 +160,7 @@ class Default(protocol.RSEProtocol):
 
             :raises RSEAccessDenied
         """
+        credentials = credentials or {}
         try:
             parse_url = urlparse(self.path2pfn(''))
             self.server = f'{parse_url.scheme}://{parse_url.netloc}'
@@ -185,12 +185,15 @@ class Default(protocol.RSEProtocol):
                 # Trying to get the proxy from the default location
                 proxy_path = '/tmp/x509up_u%s' % os.geteuid()
                 if os.path.isfile(proxy_path):
-                    x509 = proxy_path
+                    self.cert = (proxy_path, proxy_path)
                 elif self.auth_token:
+                    # If no proxy is found, we set the cert to None and use the auth_token
+                    self.cert = None
                     pass
                 else:
                     raise exception.RSEAccessDenied('X509_USER_PROXY is not set')
-            self.cert = (x509, x509)
+            else:
+                self.cert = (x509, x509)
 
         try:
             self.timeout = credentials['timeout']
@@ -332,13 +335,13 @@ class Default(protocol.RSEProtocol):
                         raise exception.RucioException(result.status_code, result.text)
                 except requests.exceptions.ConnectionError as error:
                     raise exception.ServiceUnavailable(error)
-                except IOError as error:
+                except OSError as error:
                     raise exception.SourceNotFound(error)
         except requests.exceptions.ConnectionError as error:
             raise exception.ServiceUnavailable(error)
         except requests.exceptions.ReadTimeout as error:
             raise exception.ServiceUnavailable(error)
-        except IOError as error:
+        except OSError as error:
             raise exception.SourceNotFound(error)
 
     def rename(self, pfn, new_pfn):
@@ -479,7 +482,7 @@ class Default(protocol.RSEProtocol):
 
             :param path: path to file
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
             :raises RSEAccessDenied: in case of permission issue.
 
@@ -528,13 +531,13 @@ class Default(protocol.RSEProtocol):
 
         :returns: a list with dict containing 'totalsize' and 'unusedsize'
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
         """
         endpoint_basepath = self.path2pfn('')
         headers = {'Depth': '0'}
 
         try:
-            root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text)
+            root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text)  # noqa: S314
             usedsize = root[0][1][0].find('{DAV:}quota-used-bytes').text
             try:
                 unusedsize = root[0][1][0].find('{DAV:}quota-available-bytes').text

rucio/rse/protocols/xrootd.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,13 +16,20 @@ import logging
 import os
 
 from rucio.common import exception
-from rucio.common.utils import execute, PREFERRED_CHECKSUM
+from rucio.common.utils import PREFERRED_CHECKSUM, execute
 from rucio.rse.protocols import protocol
 
 
 class Default(protocol.RSEProtocol):
     """ Implementing access to RSEs using the XRootD protocol using GSI authentication."""
 
+    @property
+    def _auth_env(self):
+        if self.auth_token:
+            return f"XrdSecPROTOCOL=ztn BEARER_TOKEN='{self.auth_token}'"
+        else:
+            return 'XrdSecPROTOCOL=gsi'
+
     def __init__(self, protocol_attr, rse_settings, logger=logging.log):
         """ Initializes the object with information about the referred RSE.
 
@@ -66,7 +72,7 @@ class Default(protocol.RSEProtocol):
         self.logger(logging.DEBUG, 'xrootd.exists: pfn: {}'.format(pfn))
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.exists: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -82,11 +88,11 @@ class Default(protocol.RSEProtocol):
 
         :param path: path to file
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
 
         :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
-        self.logger(logging.DEBUG, 'xrootd.stat: path: {}'.format(path))
+        self.logger(logging.DEBUG, f'xrootd.stat: path: {path}')
         ret = {}
         chsum = None
         if path.startswith('root:'):
@@ -94,7 +100,7 @@ class Default(protocol.RSEProtocol):
 
         try:
             # xrdfs stat for getting filesize
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s stat %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} stat {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: filesize cmd: {}'.format(cmd))
             status_stat, out, err = execute(cmd)
             if status_stat == 0:
@@ -106,7 +112,7 @@ class Default(protocol.RSEProtocol):
                             break
 
             # xrdfs query checksum for getting checksum
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s query checksum %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} query checksum {path}'
             self.logger(logging.DEBUG, 'xrootd.stat: checksum cmd: {}'.format(cmd))
             status_query, out, err = execute(cmd)
             if status_query == 0:
@@ -160,7 +166,7 @@ class Default(protocol.RSEProtocol):
         if not prefix.endswith('/'):
             prefix = ''.join([prefix, '/'])
 
-        lfns = [lfns] if type(lfns) == dict else lfns
+        lfns = [lfns] if isinstance(lfns, dict) else lfns
         for lfn in lfns:
             scope, name = lfn['scope'], lfn['name']
             if 'path' in lfn and lfn['path'] is not None:
@@ -183,7 +189,7 @@ class Default(protocol.RSEProtocol):
         try:
             # The query stats call is not implemented on some xroot doors.
             # Workaround: fail, if server does not reply within 10 seconds for static config query
-            cmd = 'XrdSecPROTOCOL=gsi XRD_REQUESTTIMEOUT=10 xrdfs %s:%s query config %s:%s' % (self.hostname, self.port, self.hostname, self.port)
+            cmd = f'{self._auth_env} XRD_REQUESTTIMEOUT=10 xrdfs {self.hostname}:{self.port} query config {self.hostname}:{self.port}'
             self.logger(logging.DEBUG, 'xrootd.connect: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -206,7 +212,7 @@ class Default(protocol.RSEProtocol):
         """
         self.logger(logging.DEBUG, 'xrootd.get: pfn: {}'.format(pfn))
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (pfn, dest)
+            cmd = f'{self._auth_env} xrdcp -f {pfn} {dest}'
             self.logger(logging.DEBUG, 'xrootd.get: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status == 54:
@@ -226,7 +232,7 @@ class Default(protocol.RSEProtocol):
             :param transfer_timeout: Transfer timeout (in seconds) - dummy
 
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.put: filename: {} target: {}'.format(filename, target))
@@ -237,7 +243,7 @@ class Default(protocol.RSEProtocol):
         if not os.path.exists(source_url):
             raise exception.SourceNotFound()
         try:
-            cmd = 'XrdSecPROTOCOL=gsi xrdcp -f %s %s' % (source_url, path)
+            cmd = f'{self._auth_env} xrdcp -f {source_url} {path}'
             self.logger(logging.DEBUG, 'xrootd.put: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -251,7 +257,7 @@ class Default(protocol.RSEProtocol):
 
             :param pfn: Physical file name
 
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.delete: pfn: {}'.format(pfn))
@@ -259,7 +265,7 @@ class Default(protocol.RSEProtocol):
             raise exception.SourceNotFound()
         try:
             path = self.pfn2path(pfn)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s rm %s' % (self.hostname, self.port, path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} rm {path}'
             self.logger(logging.DEBUG, 'xrootd.delete: cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:
@@ -273,7 +279,7 @@ class Default(protocol.RSEProtocol):
             :param pfn:      Current physical file name
             :param new_pfn  New physical file name
             :raises DestinationNotAccessible: if the destination storage was not accessible.
-            :raises ServiceUnavailable: if some generic error occured in the library.
+            :raises ServiceUnavailable: if some generic error occurred in the library.
             :raises SourceNotFound: if the source file was not found on the referred storage.
         """
         self.logger(logging.DEBUG, 'xrootd.rename: pfn: {}'.format(pfn))
@@ -283,10 +289,10 @@ class Default(protocol.RSEProtocol):
             path = self.pfn2path(pfn)
             new_path = self.pfn2path(new_pfn)
             new_dir = new_path[:new_path.rindex('/') + 1]
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mkdir -p %s' % (self.hostname, self.port, new_dir)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mkdir -p {new_dir}'
             self.logger(logging.DEBUG, 'xrootd.stat: mkdir cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
-            cmd = 'XrdSecPROTOCOL=gsi xrdfs %s:%s mv %s %s' % (self.hostname, self.port, path, new_path)
+            cmd = f'{self._auth_env} xrdfs {self.hostname}:{self.port} mv {path} {new_path}'
             self.logger(logging.DEBUG, 'xrootd.stat: rename cmd: {}'.format(cmd))
             status, out, err = execute(cmd)
             if status != 0:

rucio/rse/rsemanager.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,15 +15,28 @@
 import copy
 import logging
 import random
+from collections.abc import Callable
 from time import sleep
 from urllib.parse import urlparse
 
-from rucio.common import exception, utils, constants
-from rucio.common import types
+from rucio.common import constants, exception, types, utils
 from rucio.common.config import config_get_int
+from rucio.common.constants import RSE_SUPPORTED_PROTOCOL_OPERATIONS
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.logging import formatted_logger
-from rucio.common.utils import make_valid_did, GLOBALLY_SUPPORTED_CHECKSUMS
+from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, make_valid_did
+
+
+def get_scope_protocol(vo: str = 'def') -> Callable:
+    """
+        Returns the callable protocol to translate the pfn to a name/scope pair
+
+    :returns:
+        Callable: Scope Parser function
+    """
+    from rucio.rse.protocols.protocol import RSEDeterministicScopeTranslation
+    translation = RSEDeterministicScopeTranslation(vo=vo)
+    return translation.parser
 
 
 def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict:
@@ -118,7 +130,7 @@ def _get_possible_protocols(rse_settings: types.RSESettingsDict, operation, sche
 
 
 def get_protocols_ordered(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan', impl=None):
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():
@@ -130,7 +142,7 @@ def get_protocols_ordered(rse_settings: types.RSESettingsDict, operation, scheme
 
 
 def select_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan'):
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():
@@ -158,7 +170,7 @@ def create_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None,
 
     # Verify feasibility of Protocol
     operation = operation.lower()
-    if operation not in utils.rse_supported_protocol_operations():
+    if operation not in RSE_SUPPORTED_PROTOCOL_OPERATIONS:
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
     if domain and domain not in utils.rse_supported_protocol_domains():

rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '32.8.6',
-    'branch_nick': 'release-32-LTS',
-    'revision_id': 'ce0d5e6a899547293697ca2d6b5f6d40467c9d8e',
-    'revno': 12667
+    'version': '35.8.0',
+    'branch_nick': 'release-35-LTS',
+    'revision_id': '8a304950325b8afa9499d3acccb176ef7f93fd07',
+    'revno': 13205
 }

rucio/version.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,34 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-    from rucio.vcsversion import VERSION_INFO
-except ImportError:
-    VERSION_INFO = {'branch_nick': 'LOCALBRANCH',     # NOQA
-                    'revision_id': 'LOCALREVISION',
-                    'version': 'VERSION',
-                    'final': False,
-                    'revno': 0}
+from rucio.vcsversion import VERSION_INFO
 
 RUCIO_VERSION = [VERSION_INFO['version'], ]
 FINAL = VERSION_INFO['final']   # This becomes true at Release Candidate time
 
 
-def canonical_version_string():
+def canonical_version_string() -> str:
     """ Get the canonical string """
     return '.'.join(filter(None, RUCIO_VERSION))
 
 
-def version_string():
+def version_string() -> str:
     """ Get the version string """
     return canonical_version_string()
 
 
-def vcs_version_string():
+def vcs_version_string() -> str:
     """ Get the VCS version string """
     return "%s:%s" % (VERSION_INFO['branch_nick'], VERSION_INFO['revision_id'])
 
 
-def version_string_with_vcs():
+def version_string_with_vcs() -> str:
     """ Get the version string with VCS """
     return "%s-%s" % (canonical_version_string(), vcs_version_string())

rucio_clients-35.8.0.data/data/requirements.client.txt

@@ -0,0 +1,15 @@
+# All dependencies needed to run rucio client should be defined here
+requests>=2.32.2                                            # Python HTTP for Humans.
+urllib3>=1.26.18                                            # HTTP library with thread-safe connection pooling, file post, etc.
+dogpile-cache>=1.2.2                                        # Caching API plugins (1.1.2 is the first version to support pymemcache)
+tabulate>=0.9.0                                             # Pretty-print tabular data
+jsonschema>=4.20.0                                          # For JSON schema validation (Policy modules)
+
+# All dependencies needed in extras for rucio client should be defined here
+paramiko>=3.4.1                                             # ssh_extras; SSH2 protocol library (also needed in the server)
+kerberos>=1.3.1                                             # kerberos_extras for client and server
+pykerberos>=1.2.4                                           # kerberos_extras for client and server
+requests-kerberos>=0.14.0                                   # kerberos_extras for client and server
+python-swiftclient>=4.4.0                                   # swift_extras
+argcomplete>=3.1.6                                          # argcomplete_extras; Bash tab completion for argparse
+python-magic>=0.4.27                                        # dumper_extras; File type identification using libmagic

{rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,12 +13,11 @@
 # limitations under the License.
 
 import argparse
+import json
 import logging
-from pathlib import Path
-
 import os
-import json
 import sys
+from pathlib import Path
 
 try:
     import yaml
@@ -27,7 +25,6 @@ except ImportError:
     yaml = None
 import configparser
 
-
 # Multi-word sections used in kubernetes are slightly different from what rucio expects.
 # Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention.
 multi_word_sections = {