rucio 35.4.1__tar.gz → 35.6.0__tar.gz

{rucio-35.4.1 → rucio-35.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 35.4.1
+Version: 35.6.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-35.4.1 → rucio-35.6.0}/bin/rucio

@@ -412,7 +412,7 @@ def list_file_replicas(args):
         pfn_dir, dst_dir = args.link.split(':')
         if args.rses:
             for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
+                if replica['rses'].get(rse):
                     for pfn in replica['rses'][rse]:
                         os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
         else:
@@ -423,16 +423,17 @@ def list_file_replicas(args):
                             os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
     elif args.pfns:
         if args.rses:
-            for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
-                    for pfn in replica['rses'][rse]:
+            for replica in replicas:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if replica['rses'].get(rse):
                         print(pfn)
         else:
             for replica in replicas:
-                for rse in replica['rses']:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
                     if replica['rses'][rse]:
-                        for pfn in replica['rses'][rse]:
-                            print(pfn)
+                        print(pfn)
     else:
         if args.all_states:
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', '(STATE) RSE: REPLICA']
@@ -440,18 +441,18 @@ def list_file_replicas(args):
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']
         for replica in replicas:
             if 'bytes' in replica:
-                for rse in replica['rses']:
-                    for pfn in replica['rses'][rse]:
-                        if args.all_states:
-                            rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
-                        else:
-                            rse_string = '{0}: {1}'.format(rse, pfn)
-                        if args.rses:
-                            for selected_rse in rses:
-                                if rse == selected_rse:
-                                    table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
-                        else:
-                            table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if args.all_states:
+                        rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
+                    else:
+                        rse_string = '{0}: {1}'.format(rse, pfn)
+                    if args.rses:
+                        for selected_rse in rses:
+                            if rse == selected_rse:
+                                table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                    else:
+                        table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
         print(tabulate(table, tablefmt=tablefmt, headers=header, disable_numparse=True))
     return SUCCESS
 

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/common/exception.py

@@ -955,13 +955,22 @@ class NoDistance(RucioException):
         self.error_code = 92
 
 
-class PolicyPackageNotFound(RucioException):
+class PolicyPackageBaseException(RucioException):
+    """
+    Base exception for policy package errors.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageBaseException, self).__init__(*args)
+        self.package = package
+
+
+class PolicyPackageNotFound(PolicyPackageBaseException):
     """
     The policy package specified in the config file was not found
     """
-    def __init__(self, *args):
-        super(PolicyPackageNotFound, self).__init__(*args)
-        self._message = 'The specified policy package was not found'
+    def __init__(self, package: str, *args):
+        super(PolicyPackageNotFound, self).__init__(package, *args)
+        self._message = 'The specified policy package %s was not found' % self.package
         self.error_code = 93
 
 
@@ -1055,13 +1064,19 @@ class MetadataSchemaMismatchError(RucioException):
         self.error_code = 102
 
 
-class PolicyPackageVersionError(RucioException):
+class PolicyPackageVersionError(PolicyPackageBaseException):
     """
     Policy package is not compatible with this version of Rucio.
     """
-    def __init__(self, package, *args):
-        super(PolicyPackageVersionError, self).__init__(*args)
-        self._message = 'Policy package %s is not compatible with this Rucio version' % package
+    def __init__(self, package: str, rucio_version: str, supported_versions: list[str], *args):
+        super(PolicyPackageVersionError, self).__init__(package, *args)
+        self.rucio_version = rucio_version
+        self.supported_versions = supported_versions
+        self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
+            self.package,
+            self.rucio_version,
+            self.supported_versions
+        )
         self.error_code = 103
 
 
@@ -1096,13 +1111,13 @@ class SortingAlgorithmNotSupported(RucioException):
         self.error_code = 106
 
 
-class ErrorLoadingPolicyPackage(RucioException):
+class ErrorLoadingPolicyPackage(PolicyPackageBaseException):
     """
     An error occurred while loading the policy package.
     """
-    def __init__(self, *args):
-        super(ErrorLoadingPolicyPackage, self).__init__(*args)
-        self._message = 'An error occurred while loading the specified policy package'
+    def __init__(self, package: str, *args):
+        super(ErrorLoadingPolicyPackage, self).__init__(package, *args)
+        self._message = 'An error occurred while loading the policy package %s' % self.package
         self.error_code = 107
 
 
@@ -1114,3 +1129,23 @@ class TraceValidationSchemaNotFound(RucioException):
         super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
         self._message = 'Trace validation schema not found.'
         self.error_code = 108
+
+
+class PolicyPackageIsNotVersioned(PolicyPackageBaseException):
+    """
+    Policy package does not contain version information.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageIsNotVersioned, self).__init__(package, *args)
+        self._message = 'Policy package %s does not include information about which Rucio versions it supports.' % self.package
+        self.error_code = 109
+
+
+class UnsupportedMetadataPlugin(RucioException):
+    """
+    Raised when attempting to use a metadata plugin that is not enabled on the server.
+    """
+    def __init__(self, *args):
+        super(UnsupportedMetadataPlugin, self).__init__(*args)
+        self._message = "The requested metadata plugin is not enabled on the server."
+        self.error_code = 110

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/common/schema/__init__.py

@@ -61,9 +61,9 @@ if not multivo:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules["def"] = module
     scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
@@ -93,9 +93,9 @@ def load_schema_for_vo(vo: str) -> None:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules[vo] = module
 

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/core/did_meta_plugins/__init__.py

@@ -62,9 +62,9 @@ for meta_module_path in METADATA_PLUGIN_MODULE_PATHS:
         metadata_plugin_module = getattr(importlib.import_module(base_module), base_class)()
         METADATA_PLUGIN_MODULES.append(metadata_plugin_module)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + meta_module_path + ' not found')
+        raise exception.PolicyPackageNotFound(meta_module_path)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + meta_module_path)
+        raise exception.ErrorLoadingPolicyPackage(meta_module_path)
 
 # Set restricted character set for metadata in form character: reason
 #
@@ -84,8 +84,9 @@ def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
     :param scope: The scope of the did.
     :param name: The data identifier name.
     :param plugin: (optional) Filter specific metadata plugins.
-    :returns: List of metadata for did.
-    :raises: NotImplementedError
+    :param session: (optional) The database session in use.
+    :returns: Dictionary containing metadata for did.
+    :raises: UnsupportedMetadataPlugin: If the specified plugin is not enabled/available
     """
     if plugin.lower() == "all":
         all_metadata = {}
@@ -97,7 +98,7 @@ def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
         for metadata_plugin in METADATA_PLUGIN_MODULES:
             if metadata_plugin.get_plugin_name().lower() == plugin.lower():
                 return metadata_plugin.get_metadata(scope, name, session=session)
-    raise NotImplementedError('Metadata plugin "%s" is not enabled on the server.' % plugin)
+    raise exception.UnsupportedMetadataPlugin(f'Metadata plugin "{plugin}" is not enabled on the server.')
 
 
 @transactional_session

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/core/permission/__init__.py

@@ -69,9 +69,9 @@ if not multivo:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     permission_modules["def"] = module
 
@@ -100,9 +100,9 @@ def load_permission_for_vo(vo: str) -> None:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     permission_modules[vo] = module
 

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/core/replica_sorter.py

@@ -276,7 +276,7 @@ def sort_replicas(dictreplica: dict, client_location: dict, selection: Optional[
 
     # all sorts must be stable to preserve the priority (the Python standard sorting functions always are stable)
     if selection == 'geoip':
-        replicas = sort_geoip(dictreplica, client_location, ignore_error=True)
+        replicas = sort_geoip(dictreplica, client_location, ignore_error=config_get_bool('core', 'geoip_ignore_error', raise_exception=False, default=True))
     elif selection == 'custom_table':
         replicas = sort_custom(dictreplica, client_location)
     elif selection == 'closeness':

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/transfertool/fts3.py

@@ -1041,10 +1041,9 @@ class FTS3Transfertool(Transfertool):
             if isinstance(activity_id, int):
                 t_file['scitag'] = self.scitags_exp_id << 6 | activity_id
 
-        for plugin in self.tape_metadata_plugins:
-            plugin_hints = plugin.hints(t_file['metadata'])
-
-            t_file = deep_merge_dict(source=plugin_hints, destination=t_file)
+        if t_file['metadata']['dst_type'] == 'TAPE':
+            for plugin in self.tape_metadata_plugins:
+                t_file = deep_merge_dict(source=plugin.hints(t_file['metadata']), destination=t_file)
 
         return t_file
 

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/transfertool/fts3_plugins.py

@@ -14,14 +14,15 @@
 
 import json
 import sys
-from collections.abc import Callable
-from configparser import NoSectionError
-from typing import Any, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
-from rucio.common.config import config_get_int, config_get_items
+from rucio.common.config import config_get_int
 from rucio.common.exception import InvalidRequest
 from rucio.common.plugins import PolicyPackageAlgorithms
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 FTS3TapeMetadataPluginType = TypeVar('FTS3TapeMetadataPluginType', bound='FTS3TapeMetadataPlugin')
 
 
@@ -32,7 +33,7 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
     """
 
     ALGORITHM_NAME = "fts3_tape_metadata_plugins"
-    _HINTS_NAME = "fts3_plugins_init"
+    _INIT_FUNC_NAME = "fts3_plugins_init"
     DEFAULT = "def"
 
     def __init__(self, policy_algorithm: str) -> None:
@@ -50,22 +51,18 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
         if not self._supports(self.ALGORITHM_NAME, policy_algorithm):
             raise ValueError(f'Policy Algorithm {policy_algorithm} not found')
 
-        if self._supports(self._HINTS_NAME, policy_algorithm):
-            self._get_one_algorithm(self._HINTS_NAME, name=policy_algorithm)()
+        if self._supports(self._INIT_FUNC_NAME, policy_algorithm):
+            init_func = self._get_one_algorithm(self._INIT_FUNC_NAME, name=policy_algorithm)
+            init_func()
 
         self.set_in_hints = self._get_one_algorithm(self.ALGORITHM_NAME, name=policy_algorithm)
 
     @classmethod
     def _module_init(cls: type[FTS3TapeMetadataPluginType]) -> None:
-        cls.register(
-            "activity",
-            func=lambda x: cls._activity_hints(cls, x),  # type: ignore
-            init_func=lambda: cls._init_instance_activity_hints(cls))  # type: ignore
         cls.register(cls.DEFAULT, func=lambda x: cls._default(cls, x))  # type: ignore
-        cls.register("test", func=lambda x: cls._collocation(cls, cls._test_collocation, x))  # type: ignore
 
     @classmethod
-    def register(cls: type[FTS3TapeMetadataPluginType], name: str, func: Callable, init_func: Optional[Callable] = None) -> None:
+    def register(cls: type[FTS3TapeMetadataPluginType], name: str, func: 'Callable', init_func: Optional['Callable'] = None) -> None:
         """
         Register a fts3 transfer plugin
 
@@ -75,42 +72,18 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
         """
         super()._register(cls.ALGORITHM_NAME, algorithm_dict={name: func})
         if init_func is not None:
-            super()._register(cls._HINTS_NAME, algorithm_dict={name: init_func})
+            super()._register(cls._INIT_FUNC_NAME, algorithm_dict={name: init_func})
 
-    def _init_instance_activity_hints(self) -> None:
-        """
-            Load prorities for activities from the config
+    @staticmethod
+    def _collocation(collocation_func: 'Callable', hints: dict[str, Any]) -> dict[str, dict]:
         """
-        try:
-            self.prority_table = dict(config_get_items("tape_priority"))
-        except NoSectionError:
-            self.prority_table = {}
-
-    def _activity_hints(self, activity_kwargs: dict[str, str], default_prority: str = '20') -> dict[str, dict]:
-        """ Activity Hints - assign a priority based on activity"""
-        if "activity" in activity_kwargs:
-            activity = activity_kwargs["activity"].lower()
-
-        else:
-            raise InvalidRequest("`activity` field not found in passed metadata")
-
-        default_prority = self.prority_table.get("default", default_prority)
-        priority = self.prority_table.get(activity, default_prority)
-
-        return {"scheduling_hints": {"priority": priority}}
-
-    def _collocation(self, collocation_func: Callable, hints: dict[str, Any]) -> dict[str, dict]:
-        """
-        Wraps a 'collacation' style plugin for formatting
+        Wraps a 'collocation' style plugin for formatting
 
         :param collocation_func: Function that defines the collocation rules
         :param hints: kwargs utilized by the collocation rules
         :return: Collocation hints produced by the collocation_func, wrapped
         """
-        return {"collocation_hints": collocation_func(*hints)}
-
-    def _test_collocation(self, *hint: dict) -> dict[str, Any]:
-        return {"0": "", "1": "", "2": "", "3": ""}
+        return {"collocation_hints": collocation_func(**hints)}
 
     def _default(self, *hints: dict) -> dict:
         return {}
@@ -141,5 +114,39 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
         return {"archive_metadata": hints}
 
 
+class ActivityBasedTransferPriorityPlugin(FTS3TapeMetadataPlugin):
+    def __init__(self, policy_algorithm: str = 'activity') -> None:
+        self.register(
+            policy_algorithm,
+            func=lambda x: self._get_activity_priority(x),
+            init_func=self._init_default_priority)
+        super().__init__(policy_algorithm)
+
+    def _init_default_priority(self) -> None:
+        self.default_priority = config_get_int(
+            "tape_priority",
+            option="default",
+            raise_exception=False,
+            default=20,
+        )
+
+    def _get_activity_priority(self, activity_kwargs: dict[str, str]) -> dict[str, dict]:
+        """ Activity Hints - assign a priority based on activity"""
+        if "activity" in activity_kwargs:
+            activity = activity_kwargs["activity"]
+        else:
+            raise InvalidRequest("`activity` field not found in passed metadata")
+
+        priority = config_get_int(
+            "tape_priority",
+            option=activity,
+            raise_exception=False,
+            default=self.default_priority,
+        )
+
+        return {"scheduling_hints": {"priority": priority}}
+
+
 # Register the policies
 FTS3TapeMetadataPlugin._module_init()
+ActivityBasedTransferPriorityPlugin()

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '35.4.1',
+    'version': '35.6.0',
     'branch_nick': 'release-35-LTS',
-    'revision_id': 'b041c5990126a6987d2f884680869fc93b609acc',
-    'revno': 13162
+    'revision_id': '76a17347408868f403f602b5bf626c9744620829',
+    'revno': 13190
 }

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/web/rest/flaskapi/v1/auth.py

@@ -185,7 +185,6 @@ class UserPass(ErrorHandlingMethodView):
         headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
         headers.add('Cache-Control', 'post-check=0, pre-check=0')
         headers['Pragma'] = 'no-cache'
-
         vo = extract_vo(request.headers)
         account = request.headers.get('X-Rucio-Account', default=None)
         username = request.headers.get('X-Rucio-Username', default=None)

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/web/rest/flaskapi/v1/common.py

@@ -69,7 +69,7 @@ class CORSMiddleware:
                 response: Response = Response(status=200)
                 response.headers['Access-Control-Allow-Origin'] = request.origin
                 response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE, OPTIONS'
-                response.headers['Access-Control-Allow-Headers'] = '*'
+                response.headers['Access-Control-Allow-Headers'] = request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS')  # type: ignore (value could be None)
                 response.headers['Access-Control-Allow-Credentials'] = 'true'
                 return response(environ, start_response)
             response: Response = Response(status=403)
@@ -276,7 +276,7 @@ def generate_http_error_flask(
         exc_msg: Optional[str] = None,
         headers: Optional['HeadersType'] = None,
 ) -> "flask.Response":
-    """Utitily function to generate a complete HTTP error response.
+    """Utility function to generate a complete HTTP error response.
 
     :param status_code: The HTTP status code to generate a response for.
     :param exc: The name of the exception class or a RucioException object.

{rucio-35.4.1 → rucio-35.6.0}/lib/rucio/web/rest/flaskapi/v1/dids.py

@@ -34,6 +34,7 @@ from rucio.common.exception import (
     RSENotFound,
     RuleNotFound,
     ScopeNotFound,
+    UnsupportedMetadataPlugin,
     UnsupportedOperation,
     UnsupportedStatus,
 )
@@ -1321,6 +1322,8 @@ class Meta(ErrorHandlingMethodView):
                 schema:
                   description: A data identifier with all attributes.
                   type: object
+          400:
+            description: Bad Request - Invalid metadata plugin specified
           401:
             description: Invalid Auth Token
           404:
@@ -1339,6 +1342,8 @@ class Meta(ErrorHandlingMethodView):
             return Response(render_json(**meta), content_type='application/json')
         except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, error)
+        except UnsupportedMetadataPlugin as error:
+            return generate_http_error_flask(400, error)
 
     def post(self, scope_name):
         """

{rucio-35.4.1 → rucio-35.6.0}/tests/test_replica_sorting.py

@@ -22,6 +22,7 @@ from urllib.parse import urlparse
 import geoip2.database
 import pytest
 
+import rucio.core.config as core_config
 from rucio.common.config import config_get
 from rucio.common.constants import RseAttr
 from rucio.common.utils import parse_replicas_from_string
@@ -448,7 +449,7 @@ def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
 @pytest.mark.noparallel(reason='fails when run in parallel')
 @pytest.mark.parametrize("content_type", [Mime.METALINK, Mime.JSON_STREAM])
 def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protocols_setup, content_type):
-    """Replicas: test sorting via geoip with ignoring geoip errors."""
+    """Replicas: test sorting via geoip with ignoring geoip errors (and without)."""
 
     class MockedGeoIPError(Exception):
         def __init__(self, *args):
@@ -479,6 +480,26 @@ def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protoco
 
         get_geoip_db_mock.assert_called()
 
+    # now set config to not ignore errors
+    core_config.set("core", "geoip_ignore_error", False)
+    
+    # invalidate cache for __get_distance so that __get_geoip_db is called
+    replica_sorter.REGION.invalidate()
+
+    with mock.patch('rucio.core.replica_sorter.__geoip_db', side_effect=fake_get_geoip_db) as get_geoip_db_mock:
+        response = rest_client.post(
+            '/replicas/list',
+            headers=headers(auth(auth_token), vohdr(vo), accept(content_type)),
+            json=data
+        )
+        # this time the call should fail
+        assert response.status_code != 200
+
+        get_geoip_db_mock.assert_called()
+
+    # reset to ignore errors for other tests
+    core_config.set("core", "geoip_ignore_error", True)
+    
 
 @pytest.mark.noparallel(reason='fails when run in parallel, replicas should not be changed')
 def test_get_sorted_list_replicas_no_metalink(vo, rest_client, auth_token, protocols_setup, mock_scope):