rucio-clients 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl

rucio/common/pcache.py

@@ -44,6 +44,8 @@ DEBUG, INFO, WARN, ERROR = "DEBUG", "INFO ", "WARN ", "ERROR"
 # filename for locking
 LOCK_NAME = ".LOCK"
 
+MAXFD = 1024
+
 # Session ID
 sessid = "%s.%s" % (int(time.time()), os.getpid())
 
@@ -131,7 +133,7 @@ def unitize(x: int) -> str:
 
 class Pcache:
 
-    def Usage(self) -> None:
+    def usage(self) -> None:
         msg = """Usage: %s [flags] copy_prog [copy_flags] input output""" % self.progname
         sys.stderr.write("%s\n" % msg)  # py3, py2
     #  print>>sys.stderr, "  flags are: "
@@ -227,7 +229,7 @@ class Pcache:
             # TODO: move checksum/size validation from lsm to pcache
         except getopt.GetoptError as err:
             sys.stderr.write("%s\n" % str(err))
-            self.Usage()
+            self.usage()
             self.fail(100)
 
         for opt, arg in opts:
@@ -356,7 +358,7 @@ class Pcache:
 
         # Fail on extra args
         if not self.scratch_dir:
-            self.Usage()
+            self.usage()
             self.fail(100)
 
         # hardcoded pcache dir
@@ -417,7 +419,7 @@ class Pcache:
 
         # Fail on extra args
         if not scratch_dir:
-            self.Usage()
+            self.usage()
             self.fail(100)
 
         # If the source is lfn:, execute original command, no further action
@@ -491,7 +493,7 @@ class Pcache:
 
         # Must have a list of arguments
         if (self.parse_args(args[1:])):
-            self.Usage()
+            self.usage()
             self.fail(100)
 
         # Cache dir may have been wiped
@@ -522,7 +524,7 @@ class Pcache:
 
         # Fail on extra args
         if (len(self.args) < 3):
-            self.Usage()
+            self.usage()
             self.fail(100)
 
         self.copy_util = self.args[0]
@@ -1059,7 +1061,6 @@ class Pcache:
         os.dup2(n, i)
         os.dup2(n, o)
         os.dup2(n, e)
-        MAXFD = 1024
         try:
             import resource  # Resource usage information.
             maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

rucio/common/plugins.py

@@ -13,25 +13,73 @@
 # limitations under the License.
 
 import importlib
+import logging
 import os
-from collections.abc import Callable
 from configparser import NoOptionError, NoSectionError
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from packaging.specifiers import SpecifierSet
 
 from rucio.common import config
-from rucio.common.exception import InvalidAlgorithmName
+from rucio.common.client import get_client_vo, is_client
+from rucio.common.exception import InvalidAlgorithmName, PolicyPackageIsNotVersioned, PolicyPackageVersionError
+from rucio.version import current_version
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from rucio.common.types import LoggerFunction
 
 PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
 
 
+def check_policy_package_version(package: str, logger: 'LoggerFunction' = logging.log) -> None:
+
+    '''
+    Checks that the Rucio version supported by the policy package is compatible
+    with this version. Raises an exception if not.
+    :param package: the fully qualified name of the policy package
+    '''
+    try:
+        supported_versionset = _get_supported_versions_from_policy_package(package)
+    except ImportError:
+        logger(logging.DEBUG, 'Policy package %s not found' % package)
+        return
+    except PolicyPackageIsNotVersioned:
+        logger(logging.DEBUG, 'Policy package %s does not include information about which Rucio versions it supports' % package)
+        return
+
+    rucio_version = current_version()
+    if rucio_version not in supported_versionset:
+        raise PolicyPackageVersionError(rucio_version=rucio_version, supported_versionset=str(supported_versionset), package=package)
+
+
+def _get_supported_versions_from_policy_package(package: str) -> SpecifierSet:
+    try:
+        module = importlib.import_module(package)
+    except ImportError as e:
+        raise e
+
+    if not hasattr(module, 'SUPPORTED_VERSION'):
+        raise PolicyPackageIsNotVersioned(package)
+
+    supported_versionset = module.SUPPORTED_VERSION
+
+    if isinstance(supported_versionset, list):
+        supported_versionset = ','.join(supported_versionset)
+
+    return SpecifierSet(supported_versionset)
+
+
 class PolicyPackageAlgorithms:
     """
     Base class for Rucio Policy Package Algorithms
 
-    ALGORITHMS is of type Dict[str, Dict[str. Callable[..., Any]]]
-    where the key is the algorithm type and the value is a dictionary of algorithm names and their callables
+    ALGORITHMS is a dict where:
+        - the key is the algorithm type
+        - the value is a dictionary of algorithm names and their callables
     """
-    _ALGORITHMS: dict[str, dict[str, Callable[..., Any]]] = {}
+    _ALGORITHMS: dict[str, dict[str, 'Callable[..., Any]']] = {}
     _loaded_policy_modules = False
 
     def __init__(self) -> None:
@@ -40,14 +88,14 @@ class PolicyPackageAlgorithms:
             self._loaded_policy_modules = True
 
     @classmethod
-    def _get_one_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> Callable[..., Any]:
+    def _get_one_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> 'Callable[..., Any]':
         """
         Get the algorithm from the dictionary of algorithms
         """
         return cls._ALGORITHMS[algorithm_type][name]
 
     @classmethod
-    def _get_algorithms(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str) -> dict[str, Callable[..., Any]]:
+    def _get_algorithms(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str) -> dict[str, 'Callable[..., Any]']:
         """
         Get the dictionary of algorithms for a given type
         """
@@ -56,7 +104,7 @@ class PolicyPackageAlgorithms:
     @classmethod
     def _register(
             cls: type[PolicyPackageAlgorithmsT],
-            algorithm_type: str, algorithm_dict: dict[str, Callable[..., Any]]) -> None:
+            algorithm_type: str, algorithm_dict: dict[str, 'Callable[..., Any]']) -> None:
         """
         Provided a dictionary of callable function,
         and the associated algorithm type,
@@ -90,24 +138,9 @@ class PolicyPackageAlgorithms:
             # single policy package
             cls._try_importing_policy()
         else:
-            # determine whether on client or server
-            client = False
-            if 'RUCIO_CLIENT_MODE' not in os.environ:
-                if not config.config_has_section('database') and config.config_has_section('client'):
-                    client = True
-            else:
-                if os.environ['RUCIO_CLIENT_MODE']:
-                    client = True
-
             # on client, only register algorithms for selected VO
-            if client:
-                if 'RUCIO_VO' in os.environ:
-                    vo = os.environ['RUCIO_VO']
-                else:
-                    try:
-                        vo = str(config.config_get('client', 'vo'))
-                    except (NoOptionError, NoSectionError):
-                        vo = 'def'
+            if is_client():
+                vo = get_client_vo()
                 cls._try_importing_policy(vo)
             # on server, list all VOs and register their algorithms
             else:
@@ -121,7 +154,6 @@ class PolicyPackageAlgorithms:
     def _try_importing_policy(cls: type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
         try:
             # import from utils here to avoid circular import
-            from rucio.common.utils import check_policy_package_version
 
             env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
             package = getattr(os.environ, env_name, "")

rucio/common/policy.py

@@ -13,10 +13,11 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 from configparser import NoOptionError, NoSectionError
 from functools import wraps
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from dogpile.cache import make_region
 from dogpile.cache.api import NoValue
@@ -24,17 +25,25 @@ from dogpile.cache.api import NoValue
 from rucio.common.config import config_get
 from rucio.common.exception import UndefinedPolicy
 
+if TYPE_CHECKING:
+    from rucio.common.types import LoggerFunction
+
 REGION = make_region().configure('dogpile.cache.memory',
                                  expiration_time=900)
 
 
-def get_policy() -> str:
+def get_policy(logger: 'LoggerFunction' = logging.log) -> str:
     policy = REGION.get('policy')
     if isinstance(policy, NoValue):
         try:
             policy = config_get('policy', 'permission')
         except (NoOptionError, NoSectionError):
-            policy = 'atlas'
+            try:
+                policy = config_get('permission', 'policy')
+            except (NoOptionError, NoSectionError):
+                policy = 'def'
+                logger(logging.WARNING, "Policy not specified, falling back to 'def'")
+        policy = os.environ.get('POLICY', policy)
         REGION.set('policy', policy)
     return policy
 

rucio/common/schema/__init__.py

@@ -12,103 +12,153 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import importlib
+import logging
 from configparser import NoOptionError, NoSectionError
 from os import environ
 from typing import TYPE_CHECKING, Any
 
+from jsonschema import ValidationError, validate
+
 from rucio.common import config, exception
-from rucio.common.utils import check_policy_package_version
+from rucio.common.plugins import check_policy_package_version
 
 if TYPE_CHECKING:
     from types import ModuleType
 
+LOGGER = logging.getLogger('policy')
+
 # dictionary of schema modules for each VO
 schema_modules: dict[str, "ModuleType"] = {}
 
 # list of unique SCOPE_NAME_REGEXP values from all schemas
 scope_name_regexps: list[str] = []
 
-try:
-    multivo = config.config_get_bool('common', 'multi_vo', check_config_table=False)
-except (NoOptionError, NoSectionError):
-    multivo = False
+
+# cached function to check for multivo
+@functools.cache
+def _is_multivo():
+    try:
+        return config.config_get_bool('common', 'multi_vo', check_config_table=False)
+    except (NoOptionError, NoSectionError):
+        return False
+
+
+# cached function to get generic schema module
+@functools.cache
+def _get_generic_schema_module():
+    generic_fallback = 'generic_multi_vo' if _is_multivo() else 'generic'
+    return importlib.import_module('rucio.common.schema.' + generic_fallback)
+
 
 # multi-VO version loads schema per-VO on demand
 # we can't get a list of VOs here because the database might not
 # be available as this is imported during the bootstrapping process
-if not multivo:
+if not _is_multivo():
     GENERIC_FALLBACK = 'generic'
 
     if config.config_has_section('policy'):
         try:
             if 'RUCIO_POLICY_PACKAGE' in environ:
-                POLICY = environ['RUCIO_POLICY_PACKAGE']
+                policy = environ['RUCIO_POLICY_PACKAGE']
             else:
-                POLICY = config.config_get('policy', 'package', check_config_table=False)
-            check_policy_package_version(POLICY)
-            POLICY = POLICY + ".schema"
+                policy = config.config_get('policy', 'package', check_config_table=False)
+            check_policy_package_version(policy)
+            policy = policy + ".schema"
         except (NoOptionError, NoSectionError):
             # fall back to old system for now
             try:
-                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+                policy = config.config_get('policy', 'schema', check_config_table=False)
             except (NoOptionError, NoSectionError):
-                POLICY = GENERIC_FALLBACK
-            POLICY = 'rucio.common.schema.' + POLICY.lower()
+                policy = GENERIC_FALLBACK
+            policy = 'rucio.common.schema.' + policy.lower()
     else:
-        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+        policy = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
 
     try:
-        module = importlib.import_module(POLICY)
+        module = importlib.import_module(policy)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound(POLICY)
+        # if policy package does not contain schema module, load fallback module instead
+        # this allows a policy package to omit modules that do not need customisation
+        try:
+            LOGGER.warning('Unable to load schema module %s from policy package, falling back to %s'
+                           % (policy, GENERIC_FALLBACK))
+            policy = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+            module = importlib.import_module(policy)
+        except ModuleNotFoundError:
+            raise exception.PolicyPackageNotFound(policy)
+        except ImportError:
+            raise exception.ErrorLoadingPolicyPackage(policy)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage(POLICY)
+        raise exception.ErrorLoadingPolicyPackage(policy)
 
     schema_modules["def"] = module
-    scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
+    if hasattr(module, 'SCOPE_NAME_REGEXP'):
+        scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
 
 
 def load_schema_for_vo(vo: str) -> None:
-    GENERIC_FALLBACK = 'generic_multi_vo'
+    generic_fallback = 'generic_multi_vo'
     if config.config_has_section('policy'):
         try:
             env_name = 'RUCIO_POLICY_PACKAGE_' + vo.upper()
             if env_name in environ:
-                POLICY = environ[env_name]
+                policy = environ[env_name]
             else:
-                POLICY = config.config_get('policy', 'package-' + vo, check_config_table=False)
-            check_policy_package_version(POLICY)
-            POLICY = POLICY + ".schema"
+                policy = config.config_get('policy', 'package-' + vo, check_config_table=False)
+            check_policy_package_version(policy)
+            policy = policy + ".schema"
         except (NoOptionError, NoSectionError):
             # fall back to old system for now
             try:
-                POLICY = config.config_get('policy', 'schema', check_config_table=False)
+                policy = config.config_get('policy', 'schema', check_config_table=False)
             except (NoOptionError, NoSectionError):
-                POLICY = GENERIC_FALLBACK
-            POLICY = 'rucio.common.schema.' + POLICY.lower()
+                policy = generic_fallback
+            policy = 'rucio.common.schema.' + policy.lower()
     else:
-        POLICY = 'rucio.common.schema.' + GENERIC_FALLBACK.lower()
+        policy = 'rucio.common.schema.' + generic_fallback.lower()
 
     try:
-        module = importlib.import_module(POLICY)
+        module = importlib.import_module(policy)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound(POLICY)
+        # if policy package does not contain schema module, load fallback module instead
+        # this allows a policy package to omit modules that do not need customisation
+        try:
+            LOGGER.warning('Unable to load schema module %s from policy package, falling back to %s'
+                           % (policy, generic_fallback))
+            policy = 'rucio.common.schema.' + generic_fallback.lower()
+            module = importlib.import_module(policy)
+        except ModuleNotFoundError:
+            raise exception.PolicyPackageNotFound(policy)
+        except ImportError:
+            raise exception.ErrorLoadingPolicyPackage(policy)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage(POLICY)
+        raise exception.ErrorLoadingPolicyPackage(policy)
 
     schema_modules[vo] = module
 
 
 def validate_schema(name: str, obj: Any, vo: str = 'def') -> None:
-    if vo not in schema_modules:
-        load_schema_for_vo(vo)
-    schema_modules[vo].validate_schema(name, obj)
+    if obj:
+        if vo not in schema_modules:
+            load_schema_for_vo(vo)
+        if hasattr(schema_modules[vo], 'SCHEMAS') and name in schema_modules[vo].SCHEMAS:
+            schema = schema_modules[vo].SCHEMAS.get(name, {})
+        else:
+            # if schema not available in VO module, fall back to generic module
+            schema = _get_generic_schema_module().SCHEMAS.get(name, {})
+        try:
+            validate(obj, schema)
+        except ValidationError as error:  # NOQA: F841
+            raise exception.InvalidObject(f'Problem validating {name}: {error}')
 
 
 def get_schema_value(key: str, vo: str = 'def') -> Any:
     if vo not in schema_modules:
         load_schema_for_vo(vo)
+    if not hasattr(schema_modules[vo], key):
+        return getattr(_get_generic_schema_module(), key)
     return getattr(schema_modules[vo], key)
 
 
@@ -120,7 +170,7 @@ def get_scope_name_regexps() -> list[str]:
         from rucio.core.vo import list_vos
         vos = list_vos()
         for vo in vos:
-            if not vo['vo'] in schema_modules:
+            if vo['vo'] not in schema_modules:
                 load_schema_for_vo(vo['vo'])
             scope_name_regexp = schema_modules[vo['vo']].SCOPE_NAME_REGEXP
             if scope_name_regexp not in scope_name_regexps:

rucio/common/schema/generic.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from jsonschema import ValidationError, validate
-
-from rucio.common.exception import InvalidObject
 
 ACCOUNT_LENGTH = 25
 
@@ -417,17 +414,3 @@ SCHEMAS = {'account': ACCOUNT,
            'cache_delete_replicas': CACHE_DELETE_REPLICAS,
            'account_attribute': ACCOUNT_ATTRIBUTE,
            'import': IMPORT}
-
-
-def validate_schema(name, obj):
-    """
-    Validate object against json schema
-
-    :param name: The json schema name.
-    :param obj: The object to validate.
-    """
-    try:
-        if obj:
-            validate(obj, SCHEMAS.get(name, {}))
-    except ValidationError as error:  # NOQA, pylint: disable=W0612
-        raise InvalidObject(f'Problem validating {name}: {error}')

rucio/common/schema/generic_multi_vo.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from jsonschema import ValidationError, validate
-
-from rucio.common.exception import InvalidObject
 
 ACCOUNT_LENGTH = 29
 
@@ -396,17 +393,3 @@ SCHEMAS = {'account': ACCOUNT,
            'account_attribute': ACCOUNT_ATTRIBUTE,
            'import': IMPORT,
            'vo': VO}
-
-
-def validate_schema(name, obj):
-    """
-    Validate object against json schema
-
-    :param name: The json schema name.
-    :param obj: The object to validate.
-    """
-    try:
-        if obj:
-            validate(obj, SCHEMAS.get(name, {}))
-    except ValidationError as error:  # NOQA, pylint: disable=W0612
-        raise InvalidObject(f'Problem validating {name}: {error}')

rucio/common/test_rucio_server.py

@@ -36,27 +36,33 @@ def get_scope_and_rses():
 
     :return: A tuple (scope, rses) for the rucio client where scope is mock/test and rses is a list or (None, [None]) if no suitable rse exists.
     """
-    cmd = "rucio list-rses --rses 'test_container_xrd=True'"
+    cmd = "rucio rse list 'test_container_xrd=True'"
     print(cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     rses = out.split()
     if len(rses) == 0:
         return None, [None]
-    return 'test', rses
+
+    scope = 'test'
+    account = 'root'
+    cmd = f"rucio scope add {scope} --account {account}"
+    _, out, err = execute(cmd)
+    print(out, err)
+    return scope, rses
 
 
 def delete_rules(did):
     # get the rules for the file
     print('Deleting rules')
-    cmd = "rucio list-rules {0} | grep {0} | cut -f1 -d\\ ".format(did)
+    cmd = "rucio rule list {0} | grep {0} | cut -f1 -d\\ ".format(did)
     print(cmd)
     exitcode, out, err = execute(cmd)
     print(out, err)
     rules = out.split()
     # delete the rules for the file
     for rule in rules:
-        cmd = "rucio delete-rule {0}".format(rule)
+        cmd = "rucio rule remove {0}".format(rule)
         print(cmd)
         exitcode, out, err = execute(cmd)
 
@@ -112,7 +118,7 @@ class TestRucioServer(unittest.TestCase):
         self.assertEqual(exitcode, 0)
 
         # List the files
-        cmd = 'rucio list-files {0}:{1}'.format(self.scope, tmp_dsn)
+        cmd = 'rucio did content list --did {0}:{1}'.format(self.scope, tmp_dsn)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out)
@@ -120,7 +126,7 @@ class TestRucioServer(unittest.TestCase):
         self.assertEqual(exitcode, 0)
 
         # List the replicas
-        cmd = 'rucio list-file-replicas {0}:{1}'.format(self.scope, tmp_dsn)
+        cmd = 'rucio replica list file {0}:{1}'.format(self.scope, tmp_dsn)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out)