rucio 35.4.0__tar.gz → 35.5.0__tar.gz

{rucio-35.4.0 → rucio-35.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 35.4.0
+Version: 35.5.0
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio-35.4.0 → rucio-35.5.0}/bin/rucio-replica-recoverer

@@ -227,11 +227,11 @@ Note that attempting the use the ``--vos`` argument when in single-VO mode will
   $ rucio-replica-recoverer --run-once --vos abc xyz
   2020-07-28 15:21:33,349 5488    WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
 ''', formatter_class=argparse.RawDescriptionHelpFormatter)  # NOQA: E501
-    parser.add_argument("--nattempts", action="store", default=5, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
-    parser.add_argument("--younger-than", action="store", default=5, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
+    parser.add_argument("--nattempts", action="store", default=5, type=int, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
+    parser.add_argument("--younger-than", action="store", default=5, type=int, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
     parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
     parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
-    parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
+    parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, type=int, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
     parser.add_argument('--json-file-name', action="store", default="/opt/rucio/etc/suspicious_replica_recoverer.json", type=str, help='Name of the json file that that contains the policies which will be used by the suspicious replica recoverer.')
     parser.add_argument('--sleep-time', action="store", default=3600, type=int, help='Concurrency control: Thread sleep time after each chunk of work.')
     parser.add_argument('--active-mode', action="store_true", default=False, help='If NOT specified, the daemon will run without taking any actions on any files. In either case, the log file will be produced normally.')

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/client/didclient.py

@@ -92,7 +92,7 @@ class DIDClient(BaseClient):
             self,
             scope: str,
             name: str,
-            did_type: Literal['FILE', 'DATASET', 'CONTAINER'],
+            did_type: Literal['DATASET', 'CONTAINER'],
             statuses: Optional["Mapping[str, Any]"] = None,
             meta: Optional["Mapping[str, Any]"] = None,
             rules: Optional["Sequence[Mapping[str, Any]]"] = None,
@@ -105,7 +105,7 @@ class DIDClient(BaseClient):
 
         :param scope: The scope name.
         :param name: The data identifier name.
-        :param did_type: The data identifier type (file|dataset|container).
+        :param did_type: The data identifier type (dataset|container).
         :param statuses: Dictionary with statuses, e.g. {'monotonic':True}.
         :param meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
         :param rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/core/subscription.py

@@ -17,7 +17,7 @@ import logging
 import re
 from configparser import NoOptionError, NoSectionError
 from json import dumps
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional
 
 from sqlalchemy import and_, delete, func, select
 from sqlalchemy.exc import IntegrityError, NoResultFound, StatementError
@@ -26,12 +26,11 @@ from sqlalchemy.orm import aliased
 from rucio.common.config import config_get_bool
 from rucio.common.exception import RucioException, SubscriptionDuplicate, SubscriptionNotFound
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import SubscriptionState
+from rucio.db.sqla.constants import RuleState, SubscriptionState
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
-    from typing import Any, Optional
 
     from sqlalchemy.orm import Session
 
@@ -46,10 +45,10 @@ def add_subscription(name: str,
                      filter_: str,
                      replication_rules: str,
                      comments: str,
-                     lifetime: "Optional[int]" = None,
-                     retroactive: "Optional[bool]" = False,
-                     dry_run: "Optional[bool]" = False,
-                     priority: "Optional[int]" = 3,
+                     lifetime: Optional[int] = None,
+                     retroactive: Optional[bool] = False,
+                     dry_run: Optional[bool] = False,
+                     priority: Optional[int] = 3,
                      *, session: "Session") -> str:
     """
     Adds a new subscription which will be verified against every new added file and dataset
@@ -151,7 +150,7 @@ def update_subscription(name: str,
         keep_history = config_get_bool('subscriptions', 'keep_history')
     except (NoOptionError, NoSectionError, RuntimeError):
         keep_history = False
-    values = {'state': SubscriptionState.UPDATED}
+    values: dict[str, Any] = {'state': SubscriptionState.UPDATED}
     if 'filter' in metadata and metadata['filter']:
         values['filter'] = dumps(metadata['filter'])
     if 'replication_rules' in metadata and metadata['replication_rules']:
@@ -212,9 +211,9 @@ def update_subscription(name: str,
 
 
 @stream_session
-def list_subscriptions(name: "Optional[str]" = None,
+def list_subscriptions(name: Optional[str] = None,
                        account: "Optional[InternalAccount]" = None,
-                       state: "Optional[SubscriptionState]" = None,
+                       state: Optional[SubscriptionState] = None,
                        *, session: "Session",
                        logger: "LoggerFunction" = logging.log) -> "Iterator[SubscriptionType]":
     """
@@ -282,7 +281,13 @@ def delete_subscription(subscription_id: str, *, session: "Session") -> None:
 
 
 @stream_session
-def list_subscription_rule_states(name=None, account=None, *, session: "Session", logger=logging.log):
+def list_subscription_rule_states(
+    name: Optional[str] = None,
+    account: Optional["InternalAccount"] = None,
+    *,
+    session: "Session",
+    logger: "LoggerFunction" = logging.log
+) -> "Iterator[tuple[InternalAccount, str, RuleState, int]]":
     """Returns a list of with the number of rules per state for a subscription.
 
     :param name:               Name of the subscription
@@ -311,7 +316,7 @@ def list_subscription_rule_states(name=None, account=None, *, session: "Session"
             )
 
         if account:
-            if '*' in account.internal:
+            if account.internal and '*' in account.internal:
                 account_str = account.internal.replace('*', '%')
                 stmt = stmt.where(
                     subscription.account.like(account_str)
@@ -332,11 +337,11 @@ def list_subscription_rule_states(name=None, account=None, *, session: "Session"
     )
 
     for row in session.execute(stmt).all():
-        yield row
+        yield row._tuple()
 
 
 @read_session
-def get_subscription_by_id(subscription_id, *, session: "Session"):
+def get_subscription_by_id(subscription_id: str, *, session: "Session") -> "SubscriptionType":
     """
     Get a specific subscription by id.
 

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/db/sqla/sautils.py

@@ -17,18 +17,26 @@ SQLAlchemy utilities
 
 '''
 
+from typing import TYPE_CHECKING, Any, Union
+
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.expression import ClauseElement, Executable
 
+if TYPE_CHECKING:
+    from sqlalchemy import Table
+    from sqlalchemy.engine.interfaces import SQLCompiler
+    from sqlalchemy.sql.schema import Column
+    from sqlalchemy.sql.selectable import Select
+
 
 class InsertFromSelect(Executable, ClauseElement):
-    def __init__(self, insert_spec, select):
+    def __init__(self, insert_spec: "Union[Table, list[Column]]", select: "Select") -> None:
         self.insert_spec = insert_spec
         self.select = select
 
 
 @compiles(InsertFromSelect)
-def visit_insert_from_select(element, compiler, **kw):
+def visit_insert_from_select(element: InsertFromSelect, compiler: "SQLCompiler", **kw: Any) -> str:
     if isinstance(element.insert_spec, list):
         columns = []
         for column in element.insert_spec:

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/transfertool/fts3.py

@@ -1041,10 +1041,9 @@ class FTS3Transfertool(Transfertool):
             if isinstance(activity_id, int):
                 t_file['scitag'] = self.scitags_exp_id << 6 | activity_id
 
-        for plugin in self.tape_metadata_plugins:
-            plugin_hints = plugin.hints(t_file['metadata'])
-
-            t_file = deep_merge_dict(source=plugin_hints, destination=t_file)
+        if t_file['metadata']['dst_type'] == 'TAPE':
+            for plugin in self.tape_metadata_plugins:
+                t_file = deep_merge_dict(source=plugin.hints(t_file['metadata']), destination=t_file)
 
         return t_file
 

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/transfertool/fts3_plugins.py

@@ -62,7 +62,7 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
             func=lambda x: cls._activity_hints(cls, x),  # type: ignore
             init_func=lambda: cls._init_instance_activity_hints(cls))  # type: ignore
         cls.register(cls.DEFAULT, func=lambda x: cls._default(cls, x))  # type: ignore
-        cls.register("test", func=lambda x: cls._collocation(cls, cls._test_collocation, x))  # type: ignore
+        cls.register("test", func=lambda x: cls._collocation(cls._test_collocation, x))
 
     @classmethod
     def register(cls: type[FTS3TapeMetadataPluginType], name: str, func: Callable, init_func: Optional[Callable] = None) -> None:
@@ -99,17 +99,19 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
 
         return {"scheduling_hints": {"priority": priority}}
 
-    def _collocation(self, collocation_func: Callable, hints: dict[str, Any]) -> dict[str, dict]:
+    @staticmethod
+    def _collocation(collocation_func: 'Callable', hints: dict[str, Any]) -> dict[str, dict]:
         """
-        Wraps a 'collacation' style plugin for formatting
+        Wraps a 'collocation' style plugin for formatting
 
         :param collocation_func: Function that defines the collocation rules
         :param hints: kwargs utilized by the collocation rules
         :return: Collocation hints produced by the collocation_func, wrapped
         """
-        return {"collocation_hints": collocation_func(*hints)}
+        return {"collocation_hints": collocation_func(**hints)}
 
-    def _test_collocation(self, *hint: dict) -> dict[str, Any]:
+    @staticmethod
+    def _test_collocation(**kwargs: dict) -> dict[str, Any]:
         return {"0": "", "1": "", "2": "", "3": ""}
 
     def _default(self, *hints: dict) -> dict:

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '35.4.0',
+    'version': '35.5.0',
     'branch_nick': 'release-35-LTS',
-    'revision_id': '3fb618d968f2efbe7431b303ef916d5cdf07d133',
-    'revno': 13156
+    'revision_id': 'f5e7f329eb6baed9096242ee497152cd14464b2c',
+    'revno': 13174
 }

{rucio-35.4.0 → rucio-35.5.0}/lib/rucio/web/rest/flaskapi/v1/auth.py

@@ -185,7 +185,6 @@ class UserPass(ErrorHandlingMethodView):
         headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
         headers.add('Cache-Control', 'post-check=0, pre-check=0')
         headers['Pragma'] = 'no-cache'
-
         vo = extract_vo(request.headers)
         account = request.headers.get('X-Rucio-Account', default=None)
         username = request.headers.get('X-Rucio-Username', default=None)

{rucio-35.4.0 → rucio-35.5.0}/tests/test_transfer_plugins.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from unittest.mock import MagicMock
 
 import pytest
 
@@ -31,9 +32,14 @@ MAX_POLL_WAIT_SECONDS = 60
 TEST_FTS_HOST = "https://fts:8446"
 
 
-def _make_transfer_path(did, rse_factory, root_account):
+def _make_transfer_path(
+        did,
+        rse_factory,
+        root_account,
+        dst_rse_is_tape: bool = True
+):
     _, src_rse_id = rse_factory.make_mock_rse()
-    dst_rse, dst_rse_id = rse_factory.make_mock_rse()
+    dst_rse, dst_rse_id = rse_factory.make_mock_rse(rse_type="TAPE" if dst_rse_is_tape else "DISK")
     all_rses = [src_rse_id, dst_rse_id]
     distance_core.add_distance(src_rse_id, dst_rse_id, distance=1)
 
@@ -229,8 +235,8 @@ def test_multiple_plugin_concat(file_config_mock, did_factory, rse_factory, root
     generated_collocation_hints = job_params["archive_metadata"]["collocation_hints"]
     assert expected_hints["collocation_hints"] == generated_collocation_hints
 
-    expected_schedule_hints = job_params["archive_metadata"]["scheduling_hints"]
-    assert expected_hints["scheduling_hints"] == expected_schedule_hints
+    generated_scheduling_hints = job_params["archive_metadata"]["scheduling_hints"]
+    assert expected_hints["scheduling_hints"] == generated_scheduling_hints
 
 
 @pytest.mark.parametrize("file_config_mock", [
@@ -284,3 +290,33 @@ def test_include_none_by_default(did_factory, rse_factory, root_account):
 
     job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
     assert "archive_metadata" not in job_params
+
+
+def test_no_metadata_if_dest_is_not_tape(did_factory, rse_factory, root_account):
+    """
+    If the destination RSE is not tape,
+    no tape metadata plugin should run,
+    and no archive metadata should be included.
+    """
+
+    mock_did = did_factory.random_file_did()
+    mock_metadata_plugin = MagicMock()
+    transfer_path = _make_transfer_path(mock_did, rse_factory, root_account, dst_rse_is_tape=False)
+
+    fts3_tool = FTS3Transfertool(TEST_FTS_HOST)
+
+    fts3_tool.tape_metadata_plugins = [mock_metadata_plugin]  # type: ignore
+
+    job_params = build_job_params(
+        transfer_path=transfer_path,
+        bring_online=None,
+        default_lifetime=None,
+        archive_timeout_override=None,
+        max_time_in_queue=None,
+        logger=logging.log,
+    )
+
+    job_params = fts3_tool._file_from_transfer(transfer_path[0], job_params)
+
+    assert not mock_metadata_plugin.called
+    assert "archive_metadata" not in job_params