rucio-clients 37.3.0__py3-none-any.whl → 37.5.0__py3-none-any.whl

rucio/client/ruleclient.py

@@ -54,27 +54,52 @@ class RuleClient(BaseClient):
         weight: Optional[int] = None,
     ) -> Any:
         """
-        :param dids:                       The data identifier set.
-        :param copies:                     The number of replicas.
-        :param rse_expression:             Boolean string expression to give the list of RSEs.
-        :param priority:                   Priority of the transfers.
-        :param lifetime:                   The lifetime of the replication rules (in seconds).
-        :param grouping:                   ALL -  All files will be replicated to the same RSE.
-                                           DATASET - All files in the same dataset will be replicated to the same RSE.
-                                           NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
-        :param notify:                     Notification setting for the rule (Y, N, C).
-        :param source_replica_expression:  RSE Expression for RSEs to be considered for source replicas.
-        :param activity:                   Transfer Activity to be passed to FTS.
-        :param account:                    The account owning the rule.
-        :param meta:                       Metadata, as dictionary.
-        :param ignore_availability:        Option to ignore the availability of RSEs.
-        :param purge_replicas:             When the rule gets deleted purge the associated replicas immediately.
-        :param ask_approval:               Ask for approval of this replication rule.
-        :param asynchronous:               Create rule asynchronously by judge-injector.
-        :param locked:                     If the rule is locked, it cannot be deleted.
-        :param delay_injection:
-        :param comment:                    Comment about the rule.
-        :param weight:                     If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
+        Add a replication rule.
+
+        Parameters
+        ----------
+        dids :
+            The data identifier set.
+        copies :
+            The number of replicas.
+        rse_expression :
+            Boolean string expression to give the list of RSEs.
+        priority :
+            Priority of the transfers. Default is 3.
+        lifetime :
+            The lifetime of the replication rules (in seconds).
+        grouping :
+            ALL - All files will be replicated to the same RSE.
+            DATASET - All files in the same dataset will be replicated to the same RSE.
+            NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
+            Default is 'DATASET'.
+        notify :
+            Notification setting for the rule (Y, N, C). Default is 'N'.
+        source_replica_expression :
+            RSE Expression for RSEs to be considered for source replicas.
+        activity :
+            Transfer Activity to be passed to FTS.
+        account :
+            The account owning the rule.
+        meta :
+            Metadata, as dictionary.
+        ignore_availability :
+            Option to ignore the availability of RSEs. Default is False.
+        purge_replicas :
+            When the rule gets deleted purge the associated replicas immediately. Default is False.
+        ask_approval :
+            Ask for approval of this replication rule. Default is False.
+        asynchronous :
+            Create rule asynchronously by judge-injector. Default is False.
+        locked :
+            If the rule is locked, it cannot be deleted. Default is False.
+        delay_injection :
+            Delay the rule injection.
+        comment :
+            Comment about the rule.
+        weight :
+            If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
+
         """
         path = self.RULE_BASEURL + '/'
         url = build_url(choice(self.list_hosts), path=path)
@@ -97,9 +122,17 @@ class RuleClient(BaseClient):
         """
         Deletes a replication rule and all associated locks.
 
-        :param rule_id:         The id of the rule to be deleted
-        :param purge_replicas:  Immediately delete the replicas.
-        :raises:                RuleNotFound, AccessDenied
+        Parameters
+        ----------
+        rule_id :
+            The id of the rule to be deleted.
+        purge_replicas :
+            Immediate delete the replicas
+
+        Raises
+        -------
+        RuleNotFound
+        AccessDenied
         """
 
         path = self.RULE_BASEURL + '/' + rule_id
@@ -118,8 +151,14 @@ class RuleClient(BaseClient):
         """
         Get a replication rule.
 
-        :param rule_id:  The id of the rule to be retrieved.
-        :raises:         RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            The id of the rule to be retrieved.
+
+        Raises
+        -------
+        RuleNotFound
         """
         path = self.RULE_BASEURL + '/' + rule_id
         url = build_url(choice(self.list_hosts), path=path)
@@ -132,9 +171,16 @@ class RuleClient(BaseClient):
 
     def update_replication_rule(self, rule_id: str, options: dict[str, Any]) -> Literal[True]:
         """
-        :param rule_id:   The id of the rule to be retrieved.
-        :param options:   Options dictionary.
-        :raises:          RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            The id of the rule to be retrieved.
+        options :
+            Options dictionary.
+
+        Raises
+        -------
+        RuleNotFound
         """
         path = self.RULE_BASEURL + '/' + rule_id
         url = build_url(choice(self.list_hosts), path=path)
@@ -152,10 +198,19 @@ class RuleClient(BaseClient):
         exclude_expression: Optional[str] = None
     ) -> Any:
         """
-        :param rule_id:             Rule to be reduced.
-        :param copies:              Number of copies of the new rule.
-        :param exclude_expression:  RSE Expression of RSEs to exclude.
-        :raises:                    RuleReplaceFailed, RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            The id of the rule to be reduced.
+        copies :
+            Number of copies of the new rule.
+        exclude_expression :
+            RSE Expression of RSEs to exclude.
+
+        Raises
+        -------
+        RuleNotFound
+        RuleReplaceFailed
         """
 
         path = self.RULE_BASEURL + '/' + rule_id + '/reduce'
@@ -176,10 +231,18 @@ class RuleClient(BaseClient):
         """
         Move a replication rule to another RSE and, once done, delete the original one.
 
-        :param rule_id:                    Rule to be moved.
-        :param rse_expression:             RSE expression of the new rule.
-        :param override:                   Configurations to update for the new rule.
-        :raises:                           RuleNotFound, RuleReplaceFailed
+        Parameters
+        ----------
+        rule_id :
+            Rule to be moved.
+        rse_expression :
+            RSE expression of the new rule.
+        override :
+            Configurations to update for the new rule.
+        Raises
+        -------
+        RuleNotFound
+        RuleReplaceFailed
         """
 
         path = self.RULE_BASEURL + '/' + rule_id + '/move'
@@ -197,8 +260,14 @@ class RuleClient(BaseClient):
 
     def approve_replication_rule(self, rule_id: str) -> Literal[True]:
         """
-        :param rule_id:             Rule to be approved.
-        :raises:                    RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            Rule to be approved.
+
+        Raises
+        -------
+        RuleNotFound
         """
 
         path = self.RULE_BASEURL + '/' + rule_id
@@ -212,9 +281,17 @@ class RuleClient(BaseClient):
 
     def deny_replication_rule(self, rule_id: str, reason: Optional[str] = None) -> Literal[True]:
         """
-        :param rule_id:             Rule to be denied.
-        :param reason:              Reason for denying the rule.
-        :raises:                    RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            Rule to be denied.
+        reason :
+            Reason for denying the rule.
+
+        Raises
+        -------
+        RuleNotFound
+
         """
 
         path = self.RULE_BASEURL + '/' + rule_id
@@ -235,8 +312,12 @@ class RuleClient(BaseClient):
         """
         List the rule history of a DID.
 
-        :param scope: The scope of the DID.
-        :param name: The name of the DID.
+        Parameters
+        ----------
+        scope :
+            The scope of the DID.
+        name :
+            The name of the DID.
         """
         path = '/'.join([self.RULE_BASEURL, quote_plus(scope), quote_plus(name), 'history'])
         url = build_url(choice(self.list_hosts), path=path)
@@ -250,8 +331,13 @@ class RuleClient(BaseClient):
         """
         Examine a replication rule for errors during transfer.
 
-        :param rule_id:             Rule to be denied.
-        :raises:                    RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            The rule to be denied
+        Raises
+        -------
+        RuleNotFound
         """
         path = self.RULE_BASEURL + '/' + rule_id + '/analysis'
         url = build_url(choice(self.list_hosts), path=path)
@@ -265,8 +351,13 @@ class RuleClient(BaseClient):
         """
         List details of all replica locks for a rule.
 
-        :param rule_id:             Rule to be denied.
-        :raises:                    RuleNotFound
+        Parameters
+        ----------
+        rule_id :
+            The rule to be denied
+        Raises
+        -------
+        RuleNotFound
         """
         path = self.RULE_BASEURL + '/' + rule_id + '/locks'
         url = build_url(choice(self.list_hosts), path=path)
@@ -279,9 +370,14 @@ class RuleClient(BaseClient):
     def list_replication_rules(self, filters: Optional[dict[str, Any]] = None) -> "Iterator[dict[str, Any]]":
         """
         List all replication rules which match a filter
-        :param filters: dictionary of attributes by which the rules should be filtered
-
-        :returns: True if successful, otherwise false.
+        Parameters
+        ----------
+        filers:
+            dictionary of attributes by which the rules should be filtered
+
+        Returns
+        -------
+        True if successful, otherwise false.
         """
         filters = filters or {}
         path = self.RULE_BASEURL + '/'

rucio/client/scopeclient.py

@@ -35,11 +35,23 @@ class ScopeClient(BaseClient):
         """
         Sends the request to add a new scope.
 
-        :param account: the name of the account to add the scope to.
-        :param scope: the name of the new scope.
-        :return: True if scope was created successfully.
-        :raises Duplicate: if scope already exists.
-        :raises AccountNotFound: if account doesn't exist.
+        Parameters
+        ----------
+        account :
+            The name of the account to add the scope to.
+        scope :
+            The name of the new scope.
+
+        Returns
+        -------
+            True if scope was created successfully.
+
+        Raises
+        ------
+        Duplicate
+            If scope already exists.
+        AccountNotFound
+            If account doesn't exist.
         """
 
         path = '/'.join([self.SCOPE_BASEURL, account, 'scopes', quote_plus(scope)])
@@ -55,7 +67,9 @@ class ScopeClient(BaseClient):
         """
         Sends the request to list all scopes.
 
-        :return: a list containing the names of all scopes.
+        Returns
+        -------
+        A list containing the names of all scopes.
         """
 
         path = '/'.join(['scopes/'])
@@ -72,10 +86,21 @@ class ScopeClient(BaseClient):
         """
         Sends the request to list all scopes for a rucio account.
 
-        :param account: the rucio account to list scopes for.
-        :return: a list containing the names of all scopes for a rucio account.
-        :raises AccountNotFound: if account doesn't exist.
-        :raises ScopeNotFound: if no scopes exist for account.
+        Parameters
+        ----------
+        account :
+            The rucio account to list scopes for.
+
+        Returns
+        -------
+            A list containing the names of all scopes for a rucio account.
+
+        Raises
+        ------
+        AccountNotFound
+            If account doesn't exist.
+        ScopeNotFound
+            If no scopes exist for account.
         """
 
         path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/'])

rucio/client/subscriptionclient.py

@@ -45,16 +45,28 @@ class SubscriptionClient(BaseClient):
         """
         Adds a new subscription which will be verified against every new added file and dataset
 
-        :param name: Name of the subscription
-        :param account: Account identifier
-        :param filter_: Dictionary of attributes by which the input data should be filtered
-                       **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
-        :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
-        :param comments: Comments for the subscription
-        :param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
-        :param retroactive: Flag to know if the subscription should be applied on previous data
-        :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
-        :param priority: The priority of the subscription (3 by default)
+        Parameters
+        ----------
+        name :
+            Name of the subscription
+        account :
+            Account identifier
+        filter_ :
+            Dictionary of attributes by which the input data should be filtered
+            Example: `{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}`
+        replication_rules :
+            Replication rules to be set. Dictionary with keys copies, rse_expression, weight, rse_expression
+        comments :
+            Comments for the subscription
+        lifetime :
+            Subscription's lifetime (days); False if subscription has no lifetime
+        retroactive :
+            Flag to know if the subscription should be applied on previous data
+        dry_run :
+            Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
+        priority :
+            The priority of the subscription (3 by default)
+
         """
         path = self.SUB_BASEURL + '/' + account + '/' + name
         url = build_url(choice(self.list_hosts), path=path)
@@ -82,10 +94,21 @@ class SubscriptionClient(BaseClient):
         Returns a dictionary with the subscription information :
         Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
 
-        :param name: Name of the subscription
-        :param account: Account identifier
-        :returns: Dictionary containing subscription parameter
-        :raises: exception.NotFound if subscription is not found
+        Parameters
+        ----------
+        name :
+            Name of the subscription
+        account :
+            Account identifier
+
+        Returns
+        -------
+        Dictionary with the subscription information
+
+        Raises
+        -------
+        NotFound
+            If subscription is not found
         """
         path = self.SUB_BASEURL
         if account:
@@ -120,17 +143,23 @@ class SubscriptionClient(BaseClient):
         """
         Updates a subscription
 
-        :param name: Name of the subscription
-        :param account: Account identifier
-        :param filter_: Dictionary of attributes by which the input data should be filtered
-                       **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
-        :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
-        :param comments: Comments for the subscription
-        :param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
-        :param retroactive: Flag to know if the subscription should be applied on previous data
-        :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
-        :param priority: The priority of the subscription
-        :raises: exception.NotFound if subscription is not found
+        Parameters
+        ----------
+        name : Name of the subscription
+        account : Account identifier
+        filter_ : Dictionary of attributes by which the input data should be filtered
+            Example: `{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}`
+        replication_rules : Replication rules to be set. Dictionary with keys copies, rse_expression, weight, rse_expression
+        comments : Comments for the subscription
+        lifetime : Subscription's lifetime (days); False if subscription has no lifetime
+        retroactive : Flag to know if the subscription should be applied on previous data
+        dry_run : Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
+        priority : The priority of the subscription
+
+        Raises
+        ------
+        NotFound
+            If subscription is not found
         """
         if not account:
             account = self.account
@@ -159,8 +188,12 @@ class SubscriptionClient(BaseClient):
         """
         List the associated rules of a subscription.
 
-        :param account: Account of the subscription.
-        :param name:    Name of the subscription.
+        Parameters
+        ----------
+        account :
+            Account of the subscription.
+        name :
+            Name of the subscription.
         """
 
         path = '/'.join([self.SUB_BASEURL, account, name, 'rules'])

rucio/client/touchclient.py

@@ -40,13 +40,22 @@ class TouchClient(BaseClient):
         """
         Sends a touch trace for a given file or dataset.
 
-        :param scope: the scope of the file/dataset to update.
-        :param name: the name of file/dataset to update.
-        :param rse: optional parameter if a specific replica should be touched.
-        :raises DataIdentifierNotFound: if given dids does not exist.
-        :raises RSENotFound: if rse is not None and given rse does not exist.
-        :raises UnsupportedDIDType: if type of the given DID is not FILE or DATASET.
-        :raises RucioException: if trace could not be sent successfully.
+        Parameters
+        ----------
+        scope : The scope of the file/dataset to update.
+        name : The name of file/dataset to update.
+        rse : Optional parameter if a specific replica should be touched.
+
+        Raises
+        ------
+        DataIdentifierNotFound
+            If given dids does not exist.
+        RSENotFound
+            If rse is not None and given rse does not exist.
+        UnsupportedDIDType
+            If type of the given DID is not FILE or DATASET.
+        RucioException
+            If trace could not be sent successfully.
         """
 
         trace = {}

rucio/common/constants.py

@@ -16,8 +16,6 @@ import enum
 from collections import namedtuple
 from typing import Literal, get_args
 
-from rucio.common.config import config_get_bool
-
 """
 Constants.
 
@@ -32,21 +30,16 @@ RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic',
 KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
 # all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
 
-SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
-              'gsiftp': ['srm', 'gsiftp'],
-              'https': ['https', 'davs', 'srm+https', 'cs3s'],
-              'davs': ['https', 'davs', 'srm+https', 'cs3s'],
-              'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
-              'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
-              'root': ['root'],
-              'scp': ['scp'],
-              'rsync': ['rsync'],
-              'rclone': ['rclone']}
-if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
-    SCHEME_MAP['srm'].append('https')
-    SCHEME_MAP['https'].append('srm')
-    SCHEME_MAP['srm'].append('davs')
-    SCHEME_MAP['davs'].append('srm')
+BASE_SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
+                   'gsiftp': ['srm', 'gsiftp'],
+                   'https': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'davs': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'root': ['root'],
+                   'scp': ['scp'],
+                   'rsync': ['rsync'],
+                   'rclone': ['rclone']}
 
 SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
 SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
@@ -76,6 +69,10 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
 
 MAX_MESSAGE_LENGTH = 4000
 
+@enum.unique
+class TransferLimitDirection(enum.Enum):
+    SOURCE = 'S'
+    DESTINATION = 'D'
 
 @enum.unique
 class SuspiciousAvailability(enum.Enum):

rucio/common/utils.py

@@ -32,7 +32,7 @@ import threading
 import time
 from collections import OrderedDict
 from enum import Enum
-from functools import wraps
+from functools import cache, wraps
 from io import StringIO
 from itertools import zip_longest
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
@@ -42,7 +42,8 @@ from xml.etree import ElementTree
 
 import requests
 
-from rucio.common.config import config_get
+from rucio.common.config import config_get, config_get_bool
+from rucio.common.constants import BASE_SCHEME_MAP
 from rucio.common.exception import DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, RucioException
 from rucio.common.extra import import_extras
 from rucio.common.plugins import PolicyPackageAlgorithms
@@ -1690,3 +1691,18 @@ def is_method_overridden(obj, base_cls, method_name):
     if getattr(type(obj), method_name, None) is getattr(base_cls, method_name, None):  # Caring for bound/unbound cases
         return False
     return True
+
+
+@cache
+def get_transfer_schemas() -> dict[str, list[str]]:
+    """
+    Extend base schema map based on SRM HTTPS compatibility.
+    """
+    scheme_map = BASE_SCHEME_MAP
+    if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
+        scheme_map['srm'].append('https')
+        scheme_map['https'].append('srm')
+        scheme_map['srm'].append('davs')
+        scheme_map['davs'].append('srm')
+
+    return scheme_map

rucio/rse/rsemanager.py

@@ -24,7 +24,7 @@ from rucio.common.checksum import GLOBALLY_SUPPORTED_CHECKSUMS
 from rucio.common.config import config_get_int
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.logging import formatted_logger
-from rucio.common.utils import make_valid_did
+from rucio.common.utils import get_transfer_schemas, make_valid_did
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -875,7 +875,7 @@ def __check_compatible_scheme(
 
     if dest_scheme == src_scheme:
         return True
-    if src_scheme in constants.SCHEME_MAP.get(dest_scheme, []):
+    if src_scheme in get_transfer_schemas().get(dest_scheme, []):
         return True
 
     return False

rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '37.3.0',
+    'version': '37.5.0',
     'branch_nick': 'release-37',
-    'revision_id': '874ee3e820d3ff64ab941ccfe8a02906b5540066',
-    'revno': 13716
+    'revision_id': '092cfe864848019a987a3a167120cf5cf7df2f61',
+    'revno': 13761
 }

{rucio_clients-37.3.0.data → rucio_clients-37.5.0.data}/data/etc/rucio.cfg.template

@@ -116,7 +116,6 @@ metrics_port = 8080
 [conveyor]
 scheme = srm,gsiftp,root,http,https
 transfertool = fts3
-ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
 cacert = /opt/rucio/etc/web/ca.crt
 usercert = /opt/rucio/tools/x509up
 

{rucio_clients-37.3.0.dist-info → rucio_clients-37.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rucio-clients
-Version: 37.3.0
+Version: 37.5.0
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio