rucio-clients 35.5.0__tar.gz → 35.6.0__tar.gz

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 35.5.0
+Version: 35.6.0
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/bin/rucio

@@ -412,7 +412,7 @@ def list_file_replicas(args):
         pfn_dir, dst_dir = args.link.split(':')
         if args.rses:
             for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
+                if replica['rses'].get(rse):
                     for pfn in replica['rses'][rse]:
                         os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
         else:
@@ -423,16 +423,17 @@ def list_file_replicas(args):
                             os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
     elif args.pfns:
         if args.rses:
-            for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
-                    for pfn in replica['rses'][rse]:
+            for replica in replicas:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if replica['rses'].get(rse):
                         print(pfn)
         else:
             for replica in replicas:
-                for rse in replica['rses']:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
                     if replica['rses'][rse]:
-                        for pfn in replica['rses'][rse]:
-                            print(pfn)
+                        print(pfn)
     else:
         if args.all_states:
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', '(STATE) RSE: REPLICA']
@@ -440,18 +441,18 @@ def list_file_replicas(args):
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']
         for replica in replicas:
             if 'bytes' in replica:
-                for rse in replica['rses']:
-                    for pfn in replica['rses'][rse]:
-                        if args.all_states:
-                            rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
-                        else:
-                            rse_string = '{0}: {1}'.format(rse, pfn)
-                        if args.rses:
-                            for selected_rse in rses:
-                                if rse == selected_rse:
-                                    table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
-                        else:
-                            table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if args.all_states:
+                        rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
+                    else:
+                        rse_string = '{0}: {1}'.format(rse, pfn)
+                    if args.rses:
+                        for selected_rse in rses:
+                            if rse == selected_rse:
+                                table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                    else:
+                        table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
         print(tabulate(table, tablefmt=tablefmt, headers=header, disable_numparse=True))
     return SUCCESS
 

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/lib/rucio/common/exception.py

@@ -955,13 +955,22 @@ class NoDistance(RucioException):
         self.error_code = 92
 
 
-class PolicyPackageNotFound(RucioException):
+class PolicyPackageBaseException(RucioException):
+    """
+    Base exception for policy package errors.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageBaseException, self).__init__(*args)
+        self.package = package
+
+
+class PolicyPackageNotFound(PolicyPackageBaseException):
     """
     The policy package specified in the config file was not found
     """
-    def __init__(self, *args):
-        super(PolicyPackageNotFound, self).__init__(*args)
-        self._message = 'The specified policy package was not found'
+    def __init__(self, package: str, *args):
+        super(PolicyPackageNotFound, self).__init__(package, *args)
+        self._message = 'The specified policy package %s was not found' % self.package
         self.error_code = 93
 
 
@@ -1055,13 +1064,19 @@ class MetadataSchemaMismatchError(RucioException):
         self.error_code = 102
 
 
-class PolicyPackageVersionError(RucioException):
+class PolicyPackageVersionError(PolicyPackageBaseException):
     """
     Policy package is not compatible with this version of Rucio.
     """
-    def __init__(self, package, *args):
-        super(PolicyPackageVersionError, self).__init__(*args)
-        self._message = 'Policy package %s is not compatible with this Rucio version' % package
+    def __init__(self, package: str, rucio_version: str, supported_versions: list[str], *args):
+        super(PolicyPackageVersionError, self).__init__(package, *args)
+        self.rucio_version = rucio_version
+        self.supported_versions = supported_versions
+        self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
+            self.package,
+            self.rucio_version,
+            self.supported_versions
+        )
         self.error_code = 103
 
 
@@ -1096,13 +1111,13 @@ class SortingAlgorithmNotSupported(RucioException):
         self.error_code = 106
 
 
-class ErrorLoadingPolicyPackage(RucioException):
+class ErrorLoadingPolicyPackage(PolicyPackageBaseException):
     """
     An error occurred while loading the policy package.
     """
-    def __init__(self, *args):
-        super(ErrorLoadingPolicyPackage, self).__init__(*args)
-        self._message = 'An error occurred while loading the specified policy package'
+    def __init__(self, package: str, *args):
+        super(ErrorLoadingPolicyPackage, self).__init__(package, *args)
+        self._message = 'An error occurred while loading the policy package %s' % self.package
         self.error_code = 107
 
 
@@ -1114,3 +1129,23 @@ class TraceValidationSchemaNotFound(RucioException):
         super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
         self._message = 'Trace validation schema not found.'
         self.error_code = 108
+
+
+class PolicyPackageIsNotVersioned(PolicyPackageBaseException):
+    """
+    Policy package does not contain version information.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageIsNotVersioned, self).__init__(package, *args)
+        self._message = 'Policy package %s does not include information about which Rucio versions it supports.' % self.package
+        self.error_code = 109
+
+
+class UnsupportedMetadataPlugin(RucioException):
+    """
+    Raised when attempting to use a metadata plugin that is not enabled on the server.
+    """
+    def __init__(self, *args):
+        super(UnsupportedMetadataPlugin, self).__init__(*args)
+        self._message = "The requested metadata plugin is not enabled on the server."
+        self.error_code = 110

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/lib/rucio/common/schema/__init__.py

@@ -61,9 +61,9 @@ if not multivo:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules["def"] = module
     scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
@@ -93,9 +93,9 @@ def load_schema_for_vo(vo: str) -> None:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules[vo] = module
 

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '35.5.0',
+    'version': '35.6.0',
     'branch_nick': 'release-35-LTS',
-    'revision_id': 'f5e7f329eb6baed9096242ee497152cd14464b2c',
-    'revno': 13174
+    'revision_id': '76a17347408868f403f602b5bf626c9744620829',
+    'revno': 13190
 }

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/tests/test_replica_sorting.py

@@ -22,6 +22,7 @@ from urllib.parse import urlparse
 import geoip2.database
 import pytest
 
+import rucio.core.config as core_config
 from rucio.common.config import config_get
 from rucio.common.constants import RseAttr
 from rucio.common.utils import parse_replicas_from_string
@@ -448,7 +449,7 @@ def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
 @pytest.mark.noparallel(reason='fails when run in parallel')
 @pytest.mark.parametrize("content_type", [Mime.METALINK, Mime.JSON_STREAM])
 def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protocols_setup, content_type):
-    """Replicas: test sorting via geoip with ignoring geoip errors."""
+    """Replicas: test sorting via geoip with ignoring geoip errors (and without)."""
 
     class MockedGeoIPError(Exception):
         def __init__(self, *args):
@@ -479,6 +480,26 @@ def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protoco
 
         get_geoip_db_mock.assert_called()
 
+    # now set config to not ignore errors
+    core_config.set("core", "geoip_ignore_error", False)
+    
+    # invalidate cache for __get_distance so that __get_geoip_db is called
+    replica_sorter.REGION.invalidate()
+
+    with mock.patch('rucio.core.replica_sorter.__geoip_db', side_effect=fake_get_geoip_db) as get_geoip_db_mock:
+        response = rest_client.post(
+            '/replicas/list',
+            headers=headers(auth(auth_token), vohdr(vo), accept(content_type)),
+            json=data
+        )
+        # this time the call should fail
+        assert response.status_code != 200
+
+        get_geoip_db_mock.assert_called()
+
+    # reset to ignore errors for other tests
+    core_config.set("core", "geoip_ignore_error", True)
+    
 
 @pytest.mark.noparallel(reason='fails when run in parallel, replicas should not be changed')
 def test_get_sorted_list_replicas_no_metalink(vo, rest_client, auth_token, protocols_setup, mock_scope):

{rucio_clients-35.5.0 → rucio_clients-35.6.0}/tests/test_transfer_plugins.py

@@ -25,6 +25,7 @@ from rucio.core.topology import Topology
 from rucio.core.transfer import ProtocolFactory, build_transfer_paths
 from rucio.db.sqla.session import get_session
 from rucio.transfertool.fts3 import FTS3Transfertool, build_job_params
+from rucio.transfertool.fts3_plugins import FTS3TapeMetadataPlugin
 
 mock_session = get_session()
 
@@ -108,7 +109,7 @@ def test_scheduling_hints(file_config_mock, did_factory, rse_factory, root_accou
     assert len(job_params["archive_metadata"].keys()) == 1
     generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
 
-    expected_scheduling_hints = {"priority": "100"}
+    expected_scheduling_hints = {"priority": 100}
     assert expected_scheduling_hints == generated_scheduling_hints
 
 
@@ -150,93 +151,105 @@ def test_activity_missing(file_config_mock, did_factory, rse_factory, root_accou
     assert "archive_metadata" in job_params
     generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
 
-    expected_scheduling_hints = {"priority": "20"}
+    expected_scheduling_hints = {"priority": 20}
     assert expected_scheduling_hints == generated_scheduling_hints
 
 
-@pytest.mark.parametrize("file_config_mock", [
-    {
-        "overrides": [
-            ("transfers", "fts3tape_metadata_plugins", "test")
-        ]
-    }
-], indirect=True)
-def test_collocation_hints(file_config_mock, did_factory, rse_factory, root_account):
-    """For a mock collocation algorithm, it can produce the 4 levels of hints required for each did"""
+class TestCollocationHints:
+    class TestCollocationPlugin(FTS3TapeMetadataPlugin):
+        def __init__(self) -> None:
+            self.register(
+                'test',
+                func=lambda x: self._test_collocation(x))
+            super().__init__('test')
 
-    mock_did = did_factory.random_file_did()
-    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+        def _test_collocation(self, hints: dict[str, str]) -> dict[str, dict]:
+            return {"collocation_hints": {"0": "", "1": "", "2": "", "3": ""}}
 
-    # Mock Transfer Tool
-    fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+    TestCollocationPlugin()
 
-    job_params = build_job_params(
-        transfer_path=transfer_path,
-        bring_online=None,
-        default_lifetime=None,
-        archive_timeout_override=None,
-        max_time_in_queue=None,
-        logger=logging.log,
-    )
-
-    # Get the job params used for each transfer
-    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
-
-    expected_collocation_hints = {
-        "collocation_hints": {
-            "0": "",
-            "1": "",
-            "2": "",
-            "3": "",
+    @pytest.mark.parametrize("file_config_mock", [
+        {
+            "overrides": [
+                ("transfers", "fts3tape_metadata_plugins", "test")
+            ]
         }
-    }
-
-    assert "archive_metadata" in job_params
-    generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
-
-    assert (
-        expected_collocation_hints["collocation_hints"] == generated_collocation_hints
-    )
-
-
-@pytest.mark.parametrize("file_config_mock", [
-    {
-        "overrides": [
-            ("transfers", "fts3tape_metadata_plugins", "activity, test")
-        ]
-    }
-], indirect=True)
-def test_multiple_plugin_concat(file_config_mock, did_factory, rse_factory, root_account):
-    """When multiple plugins are used (like priority and collocation), both logics are applied"""
+    ], indirect=True)
+    def test_collocation_hints(self, file_config_mock, did_factory, rse_factory, root_account):
+        """For a mock collocation algorithm, it can produce the 4 levels of hints required for each did"""
+
+        mock_did = did_factory.random_file_did()
+        transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+
+        # Mock Transfer Tool
+        fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+
+        job_params = build_job_params(
+            transfer_path=transfer_path,
+            bring_online=None,
+            default_lifetime=None,
+            archive_timeout_override=None,
+            max_time_in_queue=None,
+            logger=logging.log,
+        )
+
+        # Get the job params used for each transfer
+        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
 
-    mock_did = did_factory.random_file_did()
-    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+        expected_collocation_hints = {
+            "collocation_hints": {
+                "0": "",
+                "1": "",
+                "2": "",
+                "3": "",
+            }
+        }
 
-    # Mock Transfer Tool
-    fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+        assert "archive_metadata" in job_params
+        generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
 
-    job_params = build_job_params(
-        transfer_path=transfer_path,
-        bring_online=None,
-        default_lifetime=None,
-        archive_timeout_override=None,
-        max_time_in_queue=None,
-        logger=logging.log,
-    )
+        assert (
+            expected_collocation_hints["collocation_hints"] == generated_collocation_hints
+        )
 
-    # Get the job params used for each transfer
-    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
-    expected_hints = {
-        "scheduling_hints": {"priority": "20"},
-        "collocation_hints": {"0": "", "1": "", "2": "", "3": ""},
-    }
-    assert "archive_metadata" in job_params
+    @pytest.mark.parametrize("file_config_mock", [
+        {
+            "overrides": [
+                ("transfers", "fts3tape_metadata_plugins", "activity, test")
+            ]
+        }
+    ], indirect=True)
+    def test_multiple_plugin_concat(self, file_config_mock, did_factory, rse_factory, root_account):
+        """When multiple plugins are used (like priority and collocation), both logics are applied"""
+
+        mock_did = did_factory.random_file_did()
+        transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+
+        # Mock Transfer Tool
+        fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+
+        job_params = build_job_params(
+            transfer_path=transfer_path,
+            bring_online=None,
+            default_lifetime=None,
+            archive_timeout_override=None,
+            max_time_in_queue=None,
+            logger=logging.log,
+        )
+
+        # Get the job params used for each transfer
+        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+        expected_hints = {
+            "scheduling_hints": {"priority": 20},
+            "collocation_hints": {"0": "", "1": "", "2": "", "3": ""},
+        }
+        assert "archive_metadata" in job_params
 
-    generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
-    assert expected_hints["collocation_hints"] == generated_collocation_hints
+        generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
+        assert expected_hints["collocation_hints"] == generated_collocation_hints
 
-    generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
-    assert expected_hints["scheduling_hints"] == generated_scheduling_hints
+        generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
+        assert expected_hints["scheduling_hints"] == generated_scheduling_hints
 
 
 @pytest.mark.parametrize("file_config_mock", [