rucio-clients 37.5.0__py3-none-any.whl → 37.7.0__py3-none-any.whl

rucio/common/cache.py

@@ -16,8 +16,7 @@ from typing import TYPE_CHECKING, Optional
 
 from dogpile.cache.region import CacheRegion
 
-from rucio.common.client import is_client
-from rucio.common.config import config_get
+from rucio.common.config import config_get, is_client
 
 if TYPE_CHECKING:
     from collections.abc import Callable

rucio/common/client.py

@@ -17,44 +17,18 @@ import socket
 from configparser import NoOptionError, NoSectionError
 from typing import TYPE_CHECKING
 
-from rucio.common.config import config_get, config_has_section
-from rucio.common.exception import ConfigNotFound
+from rucio.common.config import config_get
+from rucio.common.constants import DEFAULT_VO
 
 if TYPE_CHECKING:
     from rucio.common.types import IPWithLocationDict
 
 
-def is_client() -> bool:
-    """"
-    Checks if the function is called from a client or from a server/daemon
-
-    :returns client_mode: True if is called from a client, False if it is called from a server/daemon
-    """
-    if 'RUCIO_CLIENT_MODE' not in os.environ:
-        try:
-            if config_has_section('database'):
-                client_mode = False
-            elif config_has_section('client'):
-                client_mode = True
-            else:
-                client_mode = False
-        except (RuntimeError, ConfigNotFound):
-            # If no configuration file is found the default value should be True
-            client_mode = True
-    else:
-        if os.environ['RUCIO_CLIENT_MODE']:
-            client_mode = True
-        else:
-            client_mode = False
-
-    return client_mode
-
-
 def get_client_vo() -> str:
     """
     Get the client VO from the environment or the configuration file.
 
-    :returns vo: The client VO as a string; default = 'def'.
+    :returns vo: The client VO as a string; default = DEFAULT_VO.
     """
     if 'RUCIO_VO' in os.environ:
         vo = os.environ['RUCIO_VO']
@@ -62,7 +36,7 @@ def get_client_vo() -> str:
         try:
             vo = str(config_get('client', 'vo'))
         except (NoOptionError, NoSectionError):
-            vo = 'def'
+            vo = DEFAULT_VO
     return vo
 
 

rucio/common/config.py

@@ -32,7 +32,6 @@ if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
 
-
 def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
     if value.lower() in ['true', 'yes', 'on']:
         return True
@@ -42,7 +41,7 @@ def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
     for conv in (int, float):
         try:
             return conv(value)
-        except:
+        except Exception:
             pass
 
     return value
@@ -58,6 +57,32 @@ def _convert_to_boolean(value: Union[str, bool]) -> bool:
     raise ValueError('Not a boolean: %s' % value)
 
 
+def is_client() -> bool:
+    """"
+    Checks if the function is called from a client or from a server/daemon
+
+    :returns client_mode: True if is called from a client, False if it is called from a server/daemon
+    """
+    if 'RUCIO_CLIENT_MODE' not in os.environ:
+        try:
+            if config_has_section('database'):
+                client_mode = False
+            elif config_has_section('client'):
+                client_mode = True
+            else:
+                client_mode = False
+        except (RuntimeError, ConfigNotFound):
+            # If no configuration file is found the default value should be True
+            client_mode = True
+    else:
+        if os.environ['RUCIO_CLIENT_MODE']:
+            client_mode = True
+        else:
+            client_mode = False
+
+    return client_mode
+
+
 @overload
 def config_get(
         section: str,
@@ -193,7 +218,6 @@ def config_get(
         return convert_type_fnc(get_config().get(section, option))
     except (configparser.NoOptionError, configparser.NoSectionError, ConfigNotFound) as err:
 
-        from rucio.common.client import is_client
         client_mode = is_client()
 
         if not client_mode and check_config_table:

rucio/common/constants.py

@@ -27,6 +27,8 @@ RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic',
 # collection_keys =
 # file_keys =
 
+DEFAULT_VO = 'def'
+
 KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
 # all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
 
@@ -69,11 +71,13 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
 
 MAX_MESSAGE_LENGTH = 4000
 
+
 @enum.unique
 class TransferLimitDirection(enum.Enum):
     SOURCE = 'S'
     DESTINATION = 'D'
 
+
 @enum.unique
 class SuspiciousAvailability(enum.Enum):
     ALL = 0
@@ -211,4 +215,4 @@ RSE_ATTRS_BOOL = Literal[
 ]
 
 SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
-SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))
+SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))

rucio/common/didtype.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 """
-DID type to represent a did and to simplify operations on it
+DID type to represent a DID and to simplify operations on it
 """
 
 from typing import Any, Union
@@ -25,7 +25,7 @@ class DID:
 
     """
     Class used to store a DID
-    Given an object did of type DID
+    Given an object DID of type DID
     scope is stored in did.scope
     name is stored in did.name
     """

rucio/common/pcache.py

@@ -66,7 +66,7 @@ def run_cmd(args: "subprocess._CMD", timeout: int = 0) -> tuple[int, Optional[by
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT)
 
-    except:
+    except Exception:
         return (-2, None)
 
     # Set the timer if a timeout value was given
@@ -620,7 +620,7 @@ class Pcache:
             f.write(self.src + '\n')
             f.close()
             self.chmod(fname, 0o666)
-        except:
+        except Exception:
             pass
 
         # Record GUID if given
@@ -631,7 +631,7 @@ class Pcache:
                 f.write(self.guid + '\n')
                 f.close()
                 self.chmod(fname, 0o666)
-            except:
+            except Exception:
                 pass
 
         # Try to transfer the file up the the number of retries allowed
@@ -857,7 +857,7 @@ class Pcache:
             self.log(ERROR, "rename %s %s", xfer_file, cache_file)
             try:
                 os.unlink(xfer_file)
-            except:
+            except Exception:
                 pass
             self.fail(104)
 
@@ -904,12 +904,12 @@ class Pcache:
                 try:
                     stat_info = os.stat(src)
                     self.log(INFO, "stat(%s) = %s", src, stat_info)
-                except:
+                except Exception:
                     self.log(INFO, "cannot stat %s", src)
                 try:
                     stat_info = os.stat(dst)
                     self.log(INFO, "stat(%s) = %s", dst, stat_info)
-                except:
+                except Exception:
                     self.log(INFO, "cannot stat %s", dst)
             return ret
 
@@ -933,7 +933,7 @@ class Pcache:
             f = open(filename, 'r')
             data = int(f.read().strip())
             f.close()
-        except:
+        except Exception:
             data = 0
         return data
 
@@ -948,9 +948,9 @@ class Pcache:
             for f in os.listdir(stats_dir):
                 try:
                     os.unlink(os.path.join(stats_dir, f))
-                except:
+                except Exception:
                     pass
-        except:
+        except Exception:
             pass
         # XXXX error handling
         pass
@@ -965,7 +965,7 @@ class Pcache:
             data = f.read()
             f.close()
             value = int(data)
-        except:
+        except Exception:
             # XXXX
             value = 0
         value += delta
@@ -974,7 +974,7 @@ class Pcache:
             f.write("%s\n" % value)
             f.close()
             self.chmod(stats_file, 0o666)
-        except:
+        except Exception:
             pass
             # XXX
         self.unlock_dir(stats_dir)
@@ -993,7 +993,7 @@ class Pcache:
             f = open(filename)
             data = f.read()
             size = int(data)
-        except:
+        except Exception:
             pass
 
         # If we could not fetch the size, do a reinventory
@@ -1034,7 +1034,7 @@ class Pcache:
             f.write("%s\n" % size)
             f.close()
             self.chmod(filename, 0o666)
-        except:
+        except Exception:
             pass  # XXXX
 
         self.unlock_file(inventory_lock)
@@ -1066,16 +1066,16 @@ class Pcache:
             maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
             if (maxfd == resource.RLIM_INFINITY):
                 maxfd = MAXFD
-        except:
+        except Exception:
             try:
                 maxfd = os.sysconf("SC_OPEN_MAX")
-            except:
+            except Exception:
                 maxfd = MAXFD  # use default value
 
         for fd in range(0, maxfd + 1):
             try:
                 os.close(fd)
-            except:
+            except Exception:
                 pass
 
     # Panda server callback functions
@@ -1103,7 +1103,7 @@ class Pcache:
                              url, retry, data, ret)
                     if ret == "True":
                         break
-                except:
+                except Exception:
                     exc, msg, tb = sys.exc_info()
                     self.log(ERROR, "post to %s, data=%s, error=%s", url, data, msg)
                 retry += 1
@@ -1181,11 +1181,6 @@ class Pcache:
             return
 
         # XXXX does this create a possible race condition?
-        if 0:
-            try:
-                os.unlink(name)
-            except:
-                pass
         status = fcntl.lockf(f, fcntl.LOCK_UN)
         f.close()
         del self.locks[name]
@@ -1196,7 +1191,7 @@ class Pcache:
             try:
                 f.close()
                 os.unlink(filename)
-            except:
+            except Exception:
                 pass
 
     # Cleanup functions
@@ -1276,7 +1271,7 @@ class Pcache:
     def cleanup_failed_transfer(self) -> None:
         try:
             os.unlink(self.pcache_dir + 'xfer')
-        except:
+        except Exception:
             pass
 
     def empty_dir(self, d: str) -> None:
@@ -1295,7 +1290,7 @@ class Pcache:
                 try:
                     guid = open(fullname).read().strip()
                     self.deleted_guids.append(guid)
-                except:
+                except Exception:
                     pass  # XXXX
             elif name == "mru" and os.path.islink(fullname):
                 try:

rucio/common/plugins.py

@@ -21,47 +21,40 @@ from typing import TYPE_CHECKING, Any, TypeVar
 from packaging.specifiers import SpecifierSet
 
 from rucio.common import config
-from rucio.common.client import get_client_vo, is_client
+from rucio.common.client import get_client_vo
 from rucio.common.exception import InvalidAlgorithmName, PolicyPackageIsNotVersioned, PolicyPackageVersionError
 from rucio.version import current_version
 
 if TYPE_CHECKING:
     from collections.abc import Callable
+    from types import ModuleType
 
     from rucio.common.types import LoggerFunction
 
 PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
 
 
-def check_policy_package_version(package: str, logger: 'LoggerFunction' = logging.log) -> None:
+def check_policy_module_version(module: 'ModuleType', logger: 'LoggerFunction' = logging.log) -> None:
 
     '''
-    Checks that the Rucio version supported by the policy package is compatible
+    Checks that the Rucio version supported by the policy module is compatible
     with this version. Raises an exception if not.
-    :param package: the fully qualified name of the policy package
+    :param module: the top level module of the policy package
     '''
     try:
-        supported_versionset = _get_supported_versions_from_policy_package(package)
-    except ImportError:
-        logger(logging.DEBUG, 'Policy package %s not found' % package)
-        return
+        supported_versionset = _get_supported_versions_from_policy_package(module)
     except PolicyPackageIsNotVersioned:
-        logger(logging.DEBUG, 'Policy package %s does not include information about which Rucio versions it supports' % package)
+        logger(logging.DEBUG, 'Policy package %s does not include information about which Rucio versions it supports' % module.__name__)
         return
 
     rucio_version = current_version()
     if rucio_version not in supported_versionset:
-        raise PolicyPackageVersionError(rucio_version=rucio_version, supported_versionset=str(supported_versionset), package=package)
-
+        raise PolicyPackageVersionError(rucio_version=rucio_version, supported_versionset=str(supported_versionset), package=module.__name__)
 
-def _get_supported_versions_from_policy_package(package: str) -> SpecifierSet:
-    try:
-        module = importlib.import_module(package)
-    except ImportError as e:
-        raise e
 
+def _get_supported_versions_from_policy_package(module: 'ModuleType') -> SpecifierSet:
     if not hasattr(module, 'SUPPORTED_VERSION'):
-        raise PolicyPackageIsNotVersioned(package)
+        raise PolicyPackageIsNotVersioned(module.__name__)
 
     supported_versionset = module.SUPPORTED_VERSION
 
@@ -139,7 +132,7 @@ class PolicyPackageAlgorithms:
             cls._try_importing_policy()
         else:
             # on client, only register algorithms for selected VO
-            if is_client():
+            if config.is_client():
                 vo = get_client_vo()
                 cls._try_importing_policy(vo)
             # on server, list all VOs and register their algorithms
@@ -160,8 +153,8 @@ class PolicyPackageAlgorithms:
             if not package:
                 package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
 
-            check_policy_package_version(package)
             module = importlib.import_module(package)
+            check_policy_module_version(module)
 
             if hasattr(module, 'get_algorithms'):
                 all_algorithms = module.get_algorithms()

rucio/common/policy.py

@@ -23,6 +23,7 @@ from dogpile.cache import make_region
 from dogpile.cache.api import NoValue
 
 from rucio.common.config import config_get
+from rucio.common.constants import DEFAULT_VO
 from rucio.common.exception import UndefinedPolicy
 
 if TYPE_CHECKING:
@@ -41,8 +42,8 @@ def get_policy(logger: 'LoggerFunction' = logging.log) -> str:
             try:
                 policy = config_get('permission', 'policy')
             except (NoOptionError, NoSectionError):
-                policy = 'def'
-                logger(logging.WARNING, "Policy not specified, falling back to 'def'")
+                policy = DEFAULT_VO
+                logger(logging.WARNING, "Policy not specified, falling back to DEFAULT_VO")
         policy = os.environ.get('POLICY', policy)
         REGION.set('policy', policy)
     return policy

rucio/common/schema/__init__.py

@@ -22,7 +22,8 @@ from typing import TYPE_CHECKING, Any
 from jsonschema import ValidationError, validate
 
 from rucio.common import config, exception
-from rucio.common.plugins import check_policy_package_version
+from rucio.common.constants import DEFAULT_VO
+from rucio.common.plugins import check_policy_module_version
 
 if TYPE_CHECKING:
     from types import ModuleType
@@ -64,9 +65,10 @@ if not _is_multivo():
                 policy = environ['RUCIO_POLICY_PACKAGE']
             else:
                 policy = config.config_get('policy', 'package', check_config_table=False)
-            check_policy_package_version(policy)
+            package_module = importlib.import_module(policy)
+            check_policy_module_version(package_module)
             policy = policy + ".schema"
-        except (NoOptionError, NoSectionError):
+        except (NoOptionError, NoSectionError, ModuleNotFoundError):
             # fall back to old system for now
             try:
                 policy = config.config_get('policy', 'schema', check_config_table=False)
@@ -93,7 +95,7 @@ if not _is_multivo():
     except ImportError:
         raise exception.ErrorLoadingPolicyPackage(policy)
 
-    schema_modules["def"] = module
+    schema_modules[DEFAULT_VO] = module
     if hasattr(module, 'SCOPE_NAME_REGEXP'):
         scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
 
@@ -107,9 +109,10 @@ def load_schema_for_vo(vo: str) -> None:
                 policy = environ[env_name]
             else:
                 policy = config.config_get('policy', 'package-' + vo, check_config_table=False)
-            check_policy_package_version(policy)
+            package_module = importlib.import_module(policy)
+            check_policy_module_version(package_module)
             policy = policy + ".schema"
-        except (NoOptionError, NoSectionError):
+        except (NoOptionError, NoSectionError, ModuleNotFoundError):
             # fall back to old system for now
             try:
                 policy = config.config_get('policy', 'schema', check_config_table=False)
@@ -139,7 +142,7 @@ def load_schema_for_vo(vo: str) -> None:
     schema_modules[vo] = module
 
 
-def validate_schema(name: str, obj: Any, vo: str = 'def') -> None:
+def validate_schema(name: str, obj: Any, vo: str = DEFAULT_VO) -> None:
     if obj:
         if vo not in schema_modules:
             load_schema_for_vo(vo)
@@ -154,7 +157,7 @@ def validate_schema(name: str, obj: Any, vo: str = 'def') -> None:
             raise exception.InvalidObject(f'Problem validating {name}: {error}')
 
 
-def get_schema_value(key: str, vo: str = 'def') -> Any:
+def get_schema_value(key: str, vo: str = DEFAULT_VO) -> Any:
     if vo not in schema_modules:
         load_schema_for_vo(vo)
     if not hasattr(schema_modules[vo], key):

rucio/common/types.py

@@ -16,6 +16,8 @@ import sys
 from collections.abc import Callable
 from os import PathLike
 
+from rucio.common.constants import DEFAULT_VO
+
 if sys.version_info < (3, 11):  # pragma: no cover
     from typing_extensions import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict, TypeGuard, Union  # noqa: UP035
     PathTypeAlias = Union[PathLike, str]
@@ -35,7 +37,7 @@ class InternalType:
     '''
     Base for Internal representations of string types
     '''
-    def __init__(self, value: Optional[str], vo: str = 'def', from_external: bool = True):
+    def __init__(self, value: Optional[str], vo: str = DEFAULT_VO, from_external: bool = True):
         if value is None:
             self.external = None
             self.internal = None
@@ -87,7 +89,7 @@ class _RepresentationCalculator:
         """
         split = internal.split('@', 1)
         if len(split) == 1:  # if cannot convert, vo is '' and this is single vo
-            vo = 'def'
+            vo = DEFAULT_VO
             external = split[0]
         else:
             vo = split[1]
@@ -104,7 +106,7 @@ class _RepresentationCalculator:
 
         :returns: internal representation
         """
-        if vo == 'def':
+        if vo == DEFAULT_VO:
             return external
         internal = '{}@{}'.format(external, vo)
         return internal
@@ -114,7 +116,7 @@ class InternalAccount(InternalType):
     '''
     Internal representation of an account
     '''
-    def __init__(self, account: Optional[str], vo: str = 'def', from_external: bool = True):
+    def __init__(self, account: Optional[str], vo: str = DEFAULT_VO, from_external: bool = True):
         super(InternalAccount, self).__init__(value=account, vo=vo, from_external=from_external)
 
 
@@ -122,7 +124,7 @@ class InternalScope(InternalType):
     '''
     Internal representation of a scope
     '''
-    def __init__(self, scope: Optional[str], vo: str = 'def', from_external: bool = True):
+    def __init__(self, scope: Optional[str], vo: str = DEFAULT_VO, from_external: bool = True):
         super(InternalScope, self).__init__(value=scope, vo=vo, from_external=from_external)
 
 

rucio/common/utils.py

@@ -961,7 +961,7 @@ def parse_did_filter_from_string_fe(
 
     :param input_string: String containing the filter options.
     :param name: DID name.
-    :param type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container.
+    :param type: The type of the DID: all(container, dataset, file), collection(dataset or container), dataset, container.
     :param omit_name: omit addition of name to filters.
     :return: list of dictionaries with each dictionary as a separate OR expression.
     """

rucio/rse/__init__.py

@@ -14,7 +14,8 @@
 
 from dogpile.cache import make_region
 
-from rucio.common.client import is_client
+from rucio.common.config import is_client
+from rucio.common.constants import DEFAULT_VO
 from rucio.rse import rsemanager
 
 if is_client():
@@ -25,7 +26,7 @@ else:
     setattr(rsemanager, 'SERVER_MODE', True)
 
 
-def get_rse_client(rse, vo='def', **kwarg):
+def get_rse_client(rse, vo=DEFAULT_VO, **kwarg):
     '''
     get_rse_client
     '''
@@ -34,7 +35,7 @@ def get_rse_client(rse, vo='def', **kwarg):
     return client.get_rse(rse)
 
 
-def get_signed_url_client(rse, service, op, url, vo='def'):
+def get_signed_url_client(rse, service, op, url, vo=DEFAULT_VO):
     '''
     get_signed_url_client
     '''
@@ -42,7 +43,7 @@ def get_signed_url_client(rse, service, op, url, vo='def'):
     return CredentialClient(vo=vo).get_signed_url(rse, service, op, url)
 
 
-def get_signed_url_server(rse, service, op, url, vo='def'):
+def get_signed_url_server(rse, service, op, url, vo=DEFAULT_VO):
     '''
     get_signed_url_server
     '''
@@ -57,7 +58,7 @@ def rse_key_generator(namespace, fn, **kwargs):
     '''
     Key generator for RSE
     '''
-    def generate_key(rse, vo='def', session=None):
+    def generate_key(rse, vo=DEFAULT_VO, session=None):
         '''
         generate_key
         '''
@@ -81,7 +82,7 @@ if rsemanager.SERVER_MODE:   # pylint:disable=no-member
     from rucio.core.rse import get_rse_id, get_rse_protocols
     from rucio.core.vo import map_vo
 
-    def tmp_rse_info(rse=None, vo='def', rse_id=None, session=None):
+    def tmp_rse_info(rse=None, vo=DEFAULT_VO, rse_id=None, session=None):
         if rse_id is None:
             # This can be called directly by client tools if they're co-located on a server
             # i.e. running rucio cli on a server and during the test suite.

rucio/rse/protocols/ngarc.py

@@ -20,7 +20,7 @@ from rucio.rse.protocols import protocol
 
 try:
     import arc  # pylint: disable=import-error
-except:
+except Exception:
     pass
 
 
@@ -59,7 +59,7 @@ class Default(protocol.RSEProtocol):
         self.cfg = arc.UserConfig()
         try:
             self.cfg.ProxyPath(os.environ['X509_USER_PROXY'])
-        except:
+        except Exception:
             pass
 
     def path2pfn(self, path):

rucio/rse/protocols/srm.py

@@ -101,7 +101,7 @@ class Default(protocol.RSEProtocol):
             # force type conversion
             try:
                 port = int(port)
-            except:
+            except Exception:
                 port = ''
 
             if self.attributes['hostname'] != hostname and\

rucio/rse/protocols/webdav.py

@@ -253,8 +253,15 @@ class Default(protocol.RSEProtocol):
             :raise  ServiceUnavailable, RSEAccessDenied
         """
         path = self.path2pfn(pfn)
+
+        using_presigned_urls = self.rse['sign_url'] is not None
+
         try:
-            result = self.session.request('HEAD', path, verify=False, timeout=self.timeout, cert=self.cert)
+            # use GET instead of HEAD for presigned urls
+            if not using_presigned_urls:
+                result = self.session.request('HEAD', path, verify=False, timeout=self.timeout, cert=self.cert)
+            else:
+                result = self.session.request('GET', path, verify=False, timeout=self.timeout, cert=self.cert)
             if result.status_code == 200:
                 return True
             elif result.status_code in [401, ]: