rucio 37.4.0__py3-none-any.whl → 37.5.0__py3-none-any.whl

rucio/client/downloadclient.py

@@ -1323,7 +1323,7 @@ class DownloadClient:
     def _resolve_one_item_dids(self, item: dict[str, Any]) -> "Iterator[dict[str, Any]]":
         """
         Resolve scopes or wildcard DIDs to lists of full did names:
-        
+
         Parameters
         ----------
         item :

rucio/client/requestclient.py

@@ -19,8 +19,8 @@ from urllib.parse import quote_plus
 from requests.status_codes import codes
 
 from rucio.client.baseclient import BaseClient, choice
+from rucio.common.constants import TransferLimitDirection
 from rucio.common.utils import build_url
-from rucio.db.sqla.constants import TransferLimitDirection
 
 if TYPE_CHECKING:
     from collections.abc import Iterator, Sequence
@@ -192,14 +192,15 @@ class RequestClient(BaseClient):
         :param strategy: defines how to handle datasets: `fifo` (each file released separately) or `grouped_fifo` (wait for the entire dataset to fit)
         :param transfers: Current number of active transfers
         :param waitings: Current number of waiting transfers
-        
+
         :returns: True if the transfer limit was deleted
         """
         path = '/'.join([self.REQUEST_BASEURL, 'transfer_limits'])
         url = build_url(choice(self.list_hosts), path=path)
-        data = dumps({'rse_expression': rse_expression, 'activity': activity, 'direction': direction.value,
-                'max_transfers': max_transfers, 'volume': volume, 'deadline': deadline, 'strategy': strategy,
-                'transfers': transfers, 'waitings': waitings})
+        data = dumps({'rse_expression': rse_expression, 'activity': activity,
+                      'direction': direction.value, 'max_transfers': max_transfers,
+                      'volume': volume, 'deadline': deadline, 'strategy': strategy,
+                      'transfers': transfers, 'waitings': waitings})
         r = self._send_request(url, type_='PUT', data=data)
 
         if r.status_code == codes.created:

rucio/common/constants.py

@@ -16,8 +16,6 @@ import enum
 from collections import namedtuple
 from typing import Literal, get_args
 
-from rucio.common.config import config_get_bool
-
 """
 Constants.
 
@@ -32,21 +30,16 @@ RESERVED_KEYS = ['scope', 'name', 'account', 'did_type', 'is_open', 'monotonic',
 KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
 # all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
 
-SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
-              'gsiftp': ['srm', 'gsiftp'],
-              'https': ['https', 'davs', 'srm+https', 'cs3s'],
-              'davs': ['https', 'davs', 'srm+https', 'cs3s'],
-              'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
-              'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
-              'root': ['root'],
-              'scp': ['scp'],
-              'rsync': ['rsync'],
-              'rclone': ['rclone']}
-if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
-    SCHEME_MAP['srm'].append('https')
-    SCHEME_MAP['https'].append('srm')
-    SCHEME_MAP['srm'].append('davs')
-    SCHEME_MAP['davs'].append('srm')
+BASE_SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
+                   'gsiftp': ['srm', 'gsiftp'],
+                   'https': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'davs': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'cs3s': ['https', 'davs', 'srm+https', 'cs3s'],
+                   'root': ['root'],
+                   'scp': ['scp'],
+                   'rsync': ['rsync'],
+                   'rclone': ['rclone']}
 
 SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
 SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
@@ -76,6 +69,10 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
 
 MAX_MESSAGE_LENGTH = 4000
 
+@enum.unique
+class TransferLimitDirection(enum.Enum):
+    SOURCE = 'S'
+    DESTINATION = 'D'
 
 @enum.unique
 class SuspiciousAvailability(enum.Enum):

rucio/common/utils.py

@@ -32,7 +32,7 @@ import threading
 import time
 from collections import OrderedDict
 from enum import Enum
-from functools import wraps
+from functools import cache, wraps
 from io import StringIO
 from itertools import zip_longest
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
@@ -42,7 +42,8 @@ from xml.etree import ElementTree
 
 import requests
 
-from rucio.common.config import config_get
+from rucio.common.config import config_get, config_get_bool
+from rucio.common.constants import BASE_SCHEME_MAP
 from rucio.common.exception import DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, RucioException
 from rucio.common.extra import import_extras
 from rucio.common.plugins import PolicyPackageAlgorithms
@@ -1690,3 +1691,18 @@ def is_method_overridden(obj, base_cls, method_name):
     if getattr(type(obj), method_name, None) is getattr(base_cls, method_name, None):  # Caring for bound/unbound cases
         return False
     return True
+
+
+@cache
+def get_transfer_schemas() -> dict[str, list[str]]:
+    """
+    Extend base schema map based on SRM HTTPS compatibility.
+    """
+    scheme_map = BASE_SCHEME_MAP
+    if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False, default=False):
+        scheme_map['srm'].append('https')
+        scheme_map['https'].append('srm')
+        scheme_map['srm'].append('davs')
+        scheme_map['davs'].append('srm')
+
+    return scheme_map

rucio/core/permission/generic.py

@@ -1126,6 +1126,7 @@ def perm_export(issuer: "InternalAccount", kwargs: dict[str, Any], *, session: "
     """
     return _is_root(issuer)
 
+
 def perm_list_transfer_limits(issuer: "InternalAccount", kwargs: dict[str, Any], *, session: "Optional[Session]" = None) -> bool:
     """
     Checks if an account can list transfer limits.
@@ -1137,6 +1138,7 @@ def perm_list_transfer_limits(issuer: "InternalAccount", kwargs: dict[str, Any],
     """
     return _is_root(issuer) or has_account_attribute(account=issuer, key='admin', session=session)
 
+
 def perm_set_transfer_limit(issuer: "InternalAccount", kwargs: dict[str, Any], *, session: "Optional[Session]" = None) -> bool:
     """
     Checks if an account can set transfer limits.
@@ -1148,6 +1150,7 @@ def perm_set_transfer_limit(issuer: "InternalAccount", kwargs: dict[str, Any], *
     """
     return _is_root(issuer) or has_account_attribute(account=issuer, key='admin', session=session)
 
+
 def perm_delete_transfer_limit(issuer: "InternalAccount", kwargs: dict[str, Any], *, session: "Optional[Session]" = None) -> bool:
     """
     Checks if an account can delete transfer limits.

rucio/core/replica.py

@@ -402,7 +402,7 @@ def __declare_bad_file_replicas(
                     if f"{pfn} Unknown replica" not in unknown_replicas:
                         unknown_replicas.append('%s %s' % (pfn, 'Unknown replica'))
             elif scope or name:
-                unknown_replicas.append(f"{(scope,name)} Unknown replica")
+                unknown_replicas.append(f"{(scope, name)} Unknown replica")
 
     if status == BadFilesStatus.BAD:
         # For BAD file, we modify the replica state, not for suspicious

rucio/core/request.py

@@ -31,7 +31,7 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.sql.expression import asc, false, func, null, true
 
 from rucio.common.config import config_get_bool, config_get_int
-from rucio.common.constants import RseAttr
+from rucio.common.constants import RseAttr, TransferLimitDirection
 from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RucioException, UnsupportedOperation
 from rucio.common.types import FilterDict, InternalAccount, InternalScope, LoggerFunction, RequestDict
 from rucio.common.utils import chunks, generate_uuid
@@ -41,7 +41,7 @@ from rucio.core.monitor import MetricManager
 from rucio.core.rse import RseCollection, RseData, get_rse_attribute, get_rse_name, get_rse_vo
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import filter_thread_work, models
-from rucio.db.sqla.constants import LockState, ReplicaState, RequestErrMsg, RequestState, RequestType, TransferLimitDirection
+from rucio.db.sqla.constants import LockState, ReplicaState, RequestErrMsg, RequestState, RequestType
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 from rucio.db.sqla.util import temp_table_mngr
 

rucio/core/transfer.py

@@ -27,11 +27,10 @@ from dogpile.cache.api import NoValue
 from sqlalchemy import select, update
 from sqlalchemy.exc import IntegrityError
 
-from rucio.common import constants
 from rucio.common.config import config_get, config_get_list
-from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr
+from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr, TransferLimitDirection
 from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation
-from rucio.common.utils import construct_non_deterministic_pfn
+from rucio.common.utils import construct_non_deterministic_pfn, get_transfer_schemas
 from rucio.core import did
 from rucio.core import message as message_core
 from rucio.core import request as request_core
@@ -40,7 +39,7 @@ from rucio.core.monitor import MetricManager
 from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import DIDType, RequestState, RequestType, TransferLimitDirection
+from rucio.db.sqla.constants import DIDType, RequestState, RequestType
 from rucio.db.sqla.session import read_session, stream_session, transactional_session
 from rucio.rse import rsemanager as rsemgr
 from rucio.transfertool.bittorrent import BittorrentTransfertool
@@ -1331,7 +1330,7 @@ def __add_compatible_schemes(schemes, allowed_schemes):
     for scheme in schemes:
         if scheme in allowed_schemes:
             return_schemes.append(scheme)
-            for scheme_map_scheme in constants.SCHEME_MAP.get(scheme, []):
+            for scheme_map_scheme in get_transfer_schemas().get(scheme, []):
                 if scheme_map_scheme not in allowed_schemes:
                     continue
                 else:

rucio/daemons/conveyor/throttler.py

@@ -26,13 +26,14 @@ from sqlalchemy import null
 
 import rucio.db.sqla.util
 from rucio.common import exception
+from rucio.common.constants import TransferLimitDirection
 from rucio.common.logging import setup_logging
 from rucio.core.monitor import MetricManager
 from rucio.core.request import get_request_stats, re_sync_all_transfer_limits, release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, reset_stale_waiting_requests, set_transfer_limit_stats
 from rucio.core.rse import RseCollection, RseData
 from rucio.core.transfer import applicable_rse_transfer_limits
 from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
-from rucio.db.sqla.constants import RequestState, TransferLimitDirection
+from rucio.db.sqla.constants import RequestState
 
 if TYPE_CHECKING:
     from collections.abc import Iterator

rucio/db/sqla/constants.py

@@ -196,9 +196,9 @@ class SubscriptionState(Enum):
     BROKEN = 'B'
 
 
-class TransferLimitDirection(Enum):
-    SOURCE = 'S'
-    DESTINATION = 'D'
+#class TransferLimitDirection(Enum):
+#    SOURCE = 'S'
+#    DESTINATION = 'D'
 
 
 class DatabaseOperationType(Enum):

rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py

@@ -19,7 +19,7 @@ import sqlalchemy as sa
 from alembic import context
 from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
 
-from rucio.db.sqla.constants import TransferLimitDirection
+from rucio.common.constants import TransferLimitDirection
 from rucio.db.sqla.types import GUID
 
 # Alembic revision identifiers

rucio/db/sqla/models.py

@@ -29,6 +29,7 @@ from sqlalchemy.types import LargeBinary
 # and it must be renamed to avoid conflicts with the policy package schema modules
 from rucio.common import schema as common_schema
 from rucio.common import utils
+from rucio.common.constants import TransferLimitDirection
 from rucio.common.types import InternalAccount, InternalScope  # noqa: TCH001 (types are needed by SQLAlchemy)
 from rucio.db.sqla.constants import (
     AccountStatus,
@@ -51,7 +52,6 @@ from rucio.db.sqla.constants import (
     RuleState,
     ScopeStatus,
     SubscriptionState,
-    TransferLimitDirection,
 )
 from rucio.db.sqla.session import BASE
 from rucio.db.sqla.types import GUID, JSON, BooleanString, InternalAccountString, InternalScopeString

rucio/gateway/request.py

@@ -19,11 +19,12 @@ Interface for the requests abstraction layer
 from typing import TYPE_CHECKING, Any, Optional
 
 from rucio.common import exception
+from rucio.common.constants import TransferLimitDirection
 from rucio.common.types import InternalAccount, InternalScope, RequestGatewayDict
 from rucio.common.utils import gateway_update_return_dict
 from rucio.core import request
 from rucio.core.rse import get_rse_id
-from rucio.db.sqla.constants import DatabaseOperationType, TransferLimitDirection
+from rucio.db.sqla.constants import DatabaseOperationType
 from rucio.db.sqla.session import db_session
 from rucio.gateway import permission
 
@@ -364,14 +365,14 @@ def set_transfer_limit(
             raise exception.AccessDenied(f'{issuer} cannot set transfer limits. {auth_result.message}')
 
         request.set_transfer_limit(rse_expression=rse_expression,
-                                activity=activity,
-                                direction=direction,
-                                max_transfers=max_transfers,
-                                volume=volume,
-                                deadline=deadline,
-                                strategy=strategy,
-                                transfers=transfers,
-                                waitings=waitings)
+                                   activity=activity,
+                                   direction=direction,
+                                   max_transfers=max_transfers,
+                                   volume=volume,
+                                   deadline=deadline,
+                                   strategy=strategy,
+                                   transfers=transfers,
+                                   waitings=waitings)
 
 
 def delete_transfer_limit(
@@ -398,6 +399,6 @@ def delete_transfer_limit(
             raise exception.AccessDenied(f'{issuer} cannot delete transfer limits. {auth_result.message}')
 
         request.delete_transfer_limit(rse_expression=rse_expression,
-                                    activity=activity,
-                                    direction=direction,
-                                    session=session)
+                                      activity=activity,
+                                      direction=direction,
+                                      session=session)

rucio/rse/rsemanager.py

@@ -24,7 +24,7 @@ from rucio.common.checksum import GLOBALLY_SUPPORTED_CHECKSUMS
 from rucio.common.config import config_get_int
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.logging import formatted_logger
-from rucio.common.utils import make_valid_did
+from rucio.common.utils import get_transfer_schemas, make_valid_did
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -875,7 +875,7 @@ def __check_compatible_scheme(
 
     if dest_scheme == src_scheme:
         return True
-    if src_scheme in constants.SCHEME_MAP.get(dest_scheme, []):
+    if src_scheme in get_transfer_schemas().get(dest_scheme, []):
         return True
 
     return False

rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '37.4.0',
+    'version': '37.5.0',
     'branch_nick': 'release-37',
-    'revision_id': '20eed71b1dd6d1d8e550966b7d40ea730a16c7d8',
-    'revno': 13754
+    'revision_id': '092cfe864848019a987a3a167120cf5cf7df2f61',
+    'revno': 13761
 }

rucio/web/rest/flaskapi/v1/accountlimits.py

@@ -35,13 +35,13 @@ class LocalAccountLimit(ErrorHandlingMethodView):
         parameters:
         - name: account
           in: path
-          description: The account for the accountlimit.
+          description: "The account for the accountlimit."
           schema:
             type: string
           style: simple
         - name: rse
           in: path
-          description: The rse for the accountlimit.
+          description: "The rse for the accountlimit."
           schema:
             type: string
           style: simple
@@ -54,20 +54,20 @@ class LocalAccountLimit(ErrorHandlingMethodView):
                 - bytes
                 properties:
                   bytes:
-                    description: The new limit in bytes.
+                    description: "The new limit in bytes."
                     type: integer
         responses:
           201:
-            description: OK
+            description: "OK"
             content:
               application/json:
                 schema:
                   type: string
                   enum: ['Created']
           401:
-            description: Invalid Auth Token
+            description: "Invalid Auth Token"
           404:
-            description: No RSE or account found for the given id.
+            description: "No RSE or account found for the given id."
         """
         parameters = json_parameters()
         bytes_param = param_get(parameters, 'bytes')
@@ -89,23 +89,23 @@ class LocalAccountLimit(ErrorHandlingMethodView):
         parameters:
         - name: account
           in: path
-          description: The account for the accountlimit.
+          description: "The account for the accountlimit."
           schema:
             type: string
           style: simple
         - name: rse
           in: path
-          description: The rse for the accountlimit.
+          description: "The rse for the accountlimit."
           schema:
             type: string
           style: simple
         responses:
           200:
-            description: OK
+            description: "OK"
           401:
-            description: Invalid Auth Token
+            description: "Invalid Auth Token"
           404:
-            description: No RSE or account found for the given id.
+            description: "No RSE or account found for the given id."
         """
         try:
             delete_local_account_limit(account=account, rse=rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
@@ -127,13 +127,13 @@ class GlobalAccountLimit(ErrorHandlingMethodView):
         parameters:
         - name: account
           in: path
-          description: The account for the accountlimit.
+          description: "The account for the accountlimit."
           schema:
             type: string
           style: simple
         - name: rse_expression
           in: path
-          description: The rse expression for the accountlimit.
+          description: "The rse expression for the accountlimit."
           schema:
             type: string
           style: simple
@@ -146,20 +146,20 @@ class GlobalAccountLimit(ErrorHandlingMethodView):
                 - bytes
                 properties:
                   bytes:
-                    description: The new limit in bytes.
+                    description: "The new limit in bytes."
                     type: integer
         responses:
           201:
-            description: OK
+            description: "OK"
             content:
               application/json:
                 schema:
                   type: string
                   enum: ['Created']
           401:
-            description: Invalid Auth Token
+            description: "Invalid Auth Token"
           404:
-            description: No RSE or account found for the given id.
+            description: "No RSE or account found for the given id."
         """
         parameters = json_parameters()
         bytes_param = param_get(parameters, 'bytes')
@@ -187,23 +187,23 @@ class GlobalAccountLimit(ErrorHandlingMethodView):
         parameters:
         - name: account
           in: path
-          description: The account for the accountlimit.
+          description: "The account for the accountlimit."
           schema:
             type: string
           style: simple
         - name: rse_expression
           in: path
-          description: The rse expression for the accountlimit.
+          description: "The rse expression for the accountlimit."
           schema:
             type: string
           style: simple
         responses:
           200:
-            description: OK
+            description: "OK"
           401:
-            description: Invalid Auth Token
+            description: "Invalid Auth Token"
           404:
-            description: No RSE or account found for the given id.
+            description: "No RSE or account found for the given id."
         """
         try:
             delete_global_account_limit(account=account, rse_expression=rse_expression, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))