rucio-clients 35.4.1__tar.gz → 35.6.0__tar.gz

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 35.4.1
+Version: 35.6.0
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/bin/rucio

@@ -412,7 +412,7 @@ def list_file_replicas(args):
         pfn_dir, dst_dir = args.link.split(':')
         if args.rses:
             for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
+                if replica['rses'].get(rse):
                     for pfn in replica['rses'][rse]:
                         os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
         else:
@@ -423,16 +423,17 @@ def list_file_replicas(args):
                             os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
     elif args.pfns:
         if args.rses:
-            for replica, rse in itertools.product(replicas, rses):
-                if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
-                    for pfn in replica['rses'][rse]:
+            for replica in replicas:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if replica['rses'].get(rse):
                         print(pfn)
         else:
             for replica in replicas:
-                for rse in replica['rses']:
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
                     if replica['rses'][rse]:
-                        for pfn in replica['rses'][rse]:
-                            print(pfn)
+                        print(pfn)
     else:
         if args.all_states:
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', '(STATE) RSE: REPLICA']
@@ -440,18 +441,18 @@ def list_file_replicas(args):
             header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']
         for replica in replicas:
             if 'bytes' in replica:
-                for rse in replica['rses']:
-                    for pfn in replica['rses'][rse]:
-                        if args.all_states:
-                            rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
-                        else:
-                            rse_string = '{0}: {1}'.format(rse, pfn)
-                        if args.rses:
-                            for selected_rse in rses:
-                                if rse == selected_rse:
-                                    table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
-                        else:
-                            table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                for pfn in replica['pfns']:
+                    rse = replica['pfns'][pfn]['rse']
+                    if args.all_states:
+                        rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
+                    else:
+                        rse_string = '{0}: {1}'.format(rse, pfn)
+                    if args.rses:
+                        for selected_rse in rses:
+                            if rse == selected_rse:
+                                table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
+                    else:
+                        table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
         print(tabulate(table, tablefmt=tablefmt, headers=header, disable_numparse=True))
     return SUCCESS
 

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/lib/rucio/common/exception.py

@@ -955,13 +955,22 @@ class NoDistance(RucioException):
         self.error_code = 92
 
 
-class PolicyPackageNotFound(RucioException):
+class PolicyPackageBaseException(RucioException):
+    """
+    Base exception for policy package errors.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageBaseException, self).__init__(*args)
+        self.package = package
+
+
+class PolicyPackageNotFound(PolicyPackageBaseException):
     """
     The policy package specified in the config file was not found
     """
-    def __init__(self, *args):
-        super(PolicyPackageNotFound, self).__init__(*args)
-        self._message = 'The specified policy package was not found'
+    def __init__(self, package: str, *args):
+        super(PolicyPackageNotFound, self).__init__(package, *args)
+        self._message = 'The specified policy package %s was not found' % self.package
         self.error_code = 93
 
 
@@ -1055,13 +1064,19 @@ class MetadataSchemaMismatchError(RucioException):
         self.error_code = 102
 
 
-class PolicyPackageVersionError(RucioException):
+class PolicyPackageVersionError(PolicyPackageBaseException):
     """
     Policy package is not compatible with this version of Rucio.
     """
-    def __init__(self, package, *args):
-        super(PolicyPackageVersionError, self).__init__(*args)
-        self._message = 'Policy package %s is not compatible with this Rucio version' % package
+    def __init__(self, package: str, rucio_version: str, supported_versions: list[str], *args):
+        super(PolicyPackageVersionError, self).__init__(package, *args)
+        self.rucio_version = rucio_version
+        self.supported_versions = supported_versions
+        self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
+            self.package,
+            self.rucio_version,
+            self.supported_versions
+        )
         self.error_code = 103
 
 
@@ -1096,13 +1111,13 @@ class SortingAlgorithmNotSupported(RucioException):
         self.error_code = 106
 
 
-class ErrorLoadingPolicyPackage(RucioException):
+class ErrorLoadingPolicyPackage(PolicyPackageBaseException):
     """
     An error occurred while loading the policy package.
     """
-    def __init__(self, *args):
-        super(ErrorLoadingPolicyPackage, self).__init__(*args)
-        self._message = 'An error occurred while loading the specified policy package'
+    def __init__(self, package: str, *args):
+        super(ErrorLoadingPolicyPackage, self).__init__(package, *args)
+        self._message = 'An error occurred while loading the policy package %s' % self.package
         self.error_code = 107
 
 
@@ -1114,3 +1129,23 @@ class TraceValidationSchemaNotFound(RucioException):
         super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
         self._message = 'Trace validation schema not found.'
         self.error_code = 108
+
+
+class PolicyPackageIsNotVersioned(PolicyPackageBaseException):
+    """
+    Policy package does not contain version information.
+    """
+    def __init__(self, package: str, *args):
+        super(PolicyPackageIsNotVersioned, self).__init__(package, *args)
+        self._message = 'Policy package %s does not include information about which Rucio versions it supports.' % self.package
+        self.error_code = 109
+
+
+class UnsupportedMetadataPlugin(RucioException):
+    """
+    Raised when attempting to use a metadata plugin that is not enabled on the server.
+    """
+    def __init__(self, *args):
+        super(UnsupportedMetadataPlugin, self).__init__(*args)
+        self._message = "The requested metadata plugin is not enabled on the server."
+        self.error_code = 110

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/lib/rucio/common/schema/__init__.py

@@ -61,9 +61,9 @@ if not multivo:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules["def"] = module
     scope_name_regexps.append(module.SCOPE_NAME_REGEXP)
@@ -93,9 +93,9 @@ def load_schema_for_vo(vo: str) -> None:
     try:
         module = importlib.import_module(POLICY)
     except ModuleNotFoundError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.ErrorLoadingPolicyPackage('An error occurred while loading module ' + POLICY)
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     schema_modules[vo] = module
 

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '35.4.1',
+    'version': '35.6.0',
     'branch_nick': 'release-35-LTS',
-    'revision_id': 'b041c5990126a6987d2f884680869fc93b609acc',
-    'revno': 13162
+    'revision_id': '76a17347408868f403f602b5bf626c9744620829',
+    'revno': 13190
 }

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/tests/test_replica_sorting.py

@@ -22,6 +22,7 @@ from urllib.parse import urlparse
 import geoip2.database
 import pytest
 
+import rucio.core.config as core_config
 from rucio.common.config import config_get
 from rucio.common.constants import RseAttr
 from rucio.common.utils import parse_replicas_from_string
@@ -448,7 +449,7 @@ def test_not_sorting_lan_replicas(vo, rest_client, auth_token, protocols_setup,
 @pytest.mark.noparallel(reason='fails when run in parallel')
 @pytest.mark.parametrize("content_type", [Mime.METALINK, Mime.JSON_STREAM])
 def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protocols_setup, content_type):
-    """Replicas: test sorting via geoip with ignoring geoip errors."""
+    """Replicas: test sorting via geoip with ignoring geoip errors (and without)."""
 
     class MockedGeoIPError(Exception):
         def __init__(self, *args):
@@ -479,6 +480,26 @@ def test_sort_geoip_address_not_found_error(vo, rest_client, auth_token, protoco
 
         get_geoip_db_mock.assert_called()
 
+    # now set config to not ignore errors
+    core_config.set("core", "geoip_ignore_error", False)
+    
+    # invalidate cache for __get_distance so that __get_geoip_db is called
+    replica_sorter.REGION.invalidate()
+
+    with mock.patch('rucio.core.replica_sorter.__geoip_db', side_effect=fake_get_geoip_db) as get_geoip_db_mock:
+        response = rest_client.post(
+            '/replicas/list',
+            headers=headers(auth(auth_token), vohdr(vo), accept(content_type)),
+            json=data
+        )
+        # this time the call should fail
+        assert response.status_code != 200
+
+        get_geoip_db_mock.assert_called()
+
+    # reset to ignore errors for other tests
+    core_config.set("core", "geoip_ignore_error", True)
+    
 
 @pytest.mark.noparallel(reason='fails when run in parallel, replicas should not be changed')
 def test_get_sorted_list_replicas_no_metalink(vo, rest_client, auth_token, protocols_setup, mock_scope):

{rucio_clients-35.4.1 → rucio_clients-35.6.0}/tests/test_transfer_plugins.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from unittest.mock import MagicMock
 
 import pytest
 
@@ -24,6 +25,7 @@ from rucio.core.topology import Topology
 from rucio.core.transfer import ProtocolFactory, build_transfer_paths
 from rucio.db.sqla.session import get_session
 from rucio.transfertool.fts3 import FTS3Transfertool, build_job_params
+from rucio.transfertool.fts3_plugins import FTS3TapeMetadataPlugin
 
 mock_session = get_session()
 
@@ -31,9 +33,14 @@ MAX_POLL_WAIT_SECONDS = 60
 TEST_FTS_HOST = "https://fts:8446"
 
 
-def _make_transfer_path(did, rse_factory, root_account):
+def _make_transfer_path(
+        did,
+        rse_factory,
+        root_account,
+        dst_rse_is_tape: bool = True
+):
     _, src_rse_id = rse_factory.make_mock_rse()
-    dst_rse, dst_rse_id = rse_factory.make_mock_rse()
+    dst_rse, dst_rse_id = rse_factory.make_mock_rse(rse_type="TAPE" if dst_rse_is_tape else "DISK")
     all_rses = [src_rse_id, dst_rse_id]
     distance_core.add_distance(src_rse_id, dst_rse_id, distance=1)
 
@@ -102,7 +109,7 @@ def test_scheduling_hints(file_config_mock, did_factory, rse_factory, root_accou
     assert len(job_params["archive_metadata"].keys()) == 1
     generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
 
-    expected_scheduling_hints = {"priority": "100"}
+    expected_scheduling_hints = {"priority": 100}
     assert expected_scheduling_hints == generated_scheduling_hints
 
 
@@ -144,65 +151,117 @@ def test_activity_missing(file_config_mock, did_factory, rse_factory, root_accou
     assert "archive_metadata" in job_params
     generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
 
-    expected_scheduling_hints = {"priority": "20"}
+    expected_scheduling_hints = {"priority": 20}
     assert expected_scheduling_hints == generated_scheduling_hints
 
 
-@pytest.mark.parametrize("file_config_mock", [
-    {
-        "overrides": [
-            ("transfers", "fts3tape_metadata_plugins", "test")
-        ]
-    }
-], indirect=True)
-def test_collocation_hints(file_config_mock, did_factory, rse_factory, root_account):
-    """For a mock collocation algorithm, it can produce the 4 levels of hints required for each did"""
+class TestCollocationHints:
+    class TestCollocationPlugin(FTS3TapeMetadataPlugin):
+        def __init__(self) -> None:
+            self.register(
+                'test',
+                func=lambda x: self._test_collocation(x))
+            super().__init__('test')
 
-    mock_did = did_factory.random_file_did()
-    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+        def _test_collocation(self, hints: dict[str, str]) -> dict[str, dict]:
+            return {"collocation_hints": {"0": "", "1": "", "2": "", "3": ""}}
 
-    # Mock Transfer Tool
-    fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+    TestCollocationPlugin()
 
-    job_params = build_job_params(
-        transfer_path=transfer_path,
-        bring_online=None,
-        default_lifetime=None,
-        archive_timeout_override=None,
-        max_time_in_queue=None,
-        logger=logging.log,
-    )
+    @pytest.mark.parametrize("file_config_mock", [
+        {
+            "overrides": [
+                ("transfers", "fts3tape_metadata_plugins", "test")
+            ]
+        }
+    ], indirect=True)
+    def test_collocation_hints(self, file_config_mock, did_factory, rse_factory, root_account):
+        """For a mock collocation algorithm, it can produce the 4 levels of hints required for each did"""
+
+        mock_did = did_factory.random_file_did()
+        transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+
+        # Mock Transfer Tool
+        fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+
+        job_params = build_job_params(
+            transfer_path=transfer_path,
+            bring_online=None,
+            default_lifetime=None,
+            archive_timeout_override=None,
+            max_time_in_queue=None,
+            logger=logging.log,
+        )
+
+        # Get the job params used for each transfer
+        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
 
-    # Get the job params used for each transfer
-    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+        expected_collocation_hints = {
+            "collocation_hints": {
+                "0": "",
+                "1": "",
+                "2": "",
+                "3": "",
+            }
+        }
+
+        assert "archive_metadata" in job_params
+        generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
+
+        assert (
+            expected_collocation_hints["collocation_hints"] == generated_collocation_hints
+        )
 
-    expected_collocation_hints = {
-        "collocation_hints": {
-            "0": "",
-            "1": "",
-            "2": "",
-            "3": "",
+    @pytest.mark.parametrize("file_config_mock", [
+        {
+            "overrides": [
+                ("transfers", "fts3tape_metadata_plugins", "activity, test")
+            ]
         }
-    }
+    ], indirect=True)
+    def test_multiple_plugin_concat(self, file_config_mock, did_factory, rse_factory, root_account):
+        """When multiple plugins are used (like priority and collocation), both logics are applied"""
+
+        mock_did = did_factory.random_file_did()
+        transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+
+        # Mock Transfer Tool
+        fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+
+        job_params = build_job_params(
+            transfer_path=transfer_path,
+            bring_online=None,
+            default_lifetime=None,
+            archive_timeout_override=None,
+            max_time_in_queue=None,
+            logger=logging.log,
+        )
+
+        # Get the job params used for each transfer
+        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+        expected_hints = {
+            "scheduling_hints": {"priority": 20},
+            "collocation_hints": {"0": "", "1": "", "2": "", "3": ""},
+        }
+        assert "archive_metadata" in job_params
 
-    assert "archive_metadata" in job_params
-    generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
+        generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
+        assert expected_hints["collocation_hints"] == generated_collocation_hints
 
-    assert (
-        expected_collocation_hints["collocation_hints"] == generated_collocation_hints
-    )
+        generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
+        assert expected_hints["scheduling_hints"] == generated_scheduling_hints
 
 
 @pytest.mark.parametrize("file_config_mock", [
     {
         "overrides": [
-            ("transfers", "fts3tape_metadata_plugins", "activity, test")
+            ("transfers", "metadata_byte_limit", "4"),
+            ("transfers", "fts3tape_metadata_plugins", "def")
+
         ]
     }
 ], indirect=True)
-def test_multiple_plugin_concat(file_config_mock, did_factory, rse_factory, root_account):
-    """When multiple plugins are used (like priority and collocation), both logics are applied"""
-
+def test_transfer_over_limit(file_config_mock, did_factory, rse_factory, root_account):
     mock_did = did_factory.random_file_did()
     transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
 
@@ -218,31 +277,15 @@ def test_multiple_plugin_concat(file_config_mock, did_factory, rse_factory, root
         logger=logging.log,
     )
 
-    # Get the job params used for each transfer
-    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
-    expected_hints = {
-        "scheduling_hints": {"priority": "20"},
-        "collocation_hints": {"0": "", "1": "", "2": "", "3": ""},
-    }
-    assert "archive_metadata" in job_params
-
-    generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
-    assert expected_hints["collocation_hints"] == generated_collocation_hints
+    from rucio.common.exception import InvalidRequest
 
-    expected_schedule_hints = job_params["archive_metadata"]["scheduling_hints"]
-    assert expected_hints["scheduling_hints"] == expected_schedule_hints
+    with pytest.raises(InvalidRequest):
+        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
 
 
-@pytest.mark.parametrize("file_config_mock", [
-    {
-        "overrides": [
-            ("transfers", "metadata_byte_limit", "4"),
-            ("transfers", "fts3tape_metadata_plugins", "def")
+def test_include_none_by_default(did_factory, rse_factory, root_account):
+    # When there is not plugin specified in the config, do not include any archive metadata
 
-        ]
-    }
-], indirect=True)
-def test_transfer_over_limit(file_config_mock, did_factory, rse_factory, root_account):
     mock_did = did_factory.random_file_did()
     transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
 
@@ -258,21 +301,25 @@ def test_transfer_over_limit(file_config_mock, did_factory, rse_factory, root_ac
         logger=logging.log,
     )
 
-    from rucio.common.exception import InvalidRequest
-
-    with pytest.raises(InvalidRequest):
-        job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+    assert "archive_metadata" not in job_params
 
 
-def test_include_none_by_default(did_factory, rse_factory, root_account):
-    # When there is not plugin specified in the config, do not include any archive metadata
+def test_no_metadata_if_dest_is_not_tape(did_factory, rse_factory, root_account):
+    """
+    If the destination RSE is not tape,
+    no tape metadata plugin should run,
+    and no archive metadata should be included.
+    """
 
     mock_did = did_factory.random_file_did()
-    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account)
+    mock_metadata_plugin = MagicMock()
+    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account, dst_rse_is_tape=False)
 
-    # Mock Transfer Tool
     fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
 
+    fts3_tool.tape_metadata_plugins = [mock_metadata_plugin]  # type: ignore
+
     job_params = build_job_params(
         transfer_path=transfer_path,
         bring_online=None,
@@ -283,4 +330,6 @@ def test_include_none_by_default(did_factory, rse_factory, root_account):
     )
 
     job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+
+    assert not mock_metadata_plugin.called
     assert "archive_metadata" not in job_params