rucio 32.6.0.post1__tar.gz → 33.0.0rc2__tar.gz

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio
-Version: 32.6.0.post1
+Version: 33.0.0rc2
 Summary: Rucio Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio
@@ -21,32 +21,32 @@ Requires-Python: >=3.9, <4
 License-File: LICENSE
 License-File: AUTHORS.rst
 Requires-Dist: requests<=2.31.0,>=2.25.1
-Requires-Dist: urllib3~=1.26.16
+Requires-Dist: urllib3~=1.26.18
 Requires-Dist: dogpile.cache~=1.2.2
 Requires-Dist: tabulate~=0.9.0
-Requires-Dist: jsonschema~=4.18.4
-Requires-Dist: paramiko~=3.2.0
-Requires-Dist: argcomplete~=3.1.1
+Requires-Dist: jsonschema~=4.20.0
+Requires-Dist: paramiko~=3.3.1
+Requires-Dist: argcomplete~=3.1.6
 Requires-Dist: python-magic~=0.4.27
-Requires-Dist: SQLAlchemy==2.0.19
-Requires-Dist: alembic~=1.11.1
+Requires-Dist: SQLAlchemy==2.0.23
+Requires-Dist: alembic~=1.12.1
 Requires-Dist: pymemcache==4.0.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: stomp.py==8.1.0
 Requires-Dist: statsd==4.0.1
 Requires-Dist: geoip2==4.7.0
-Requires-Dist: google-auth==2.22.0
-Requires-Dist: redis==4.6.0
-Requires-Dist: Flask==2.3.2
+Requires-Dist: google-auth==2.23.4
+Requires-Dist: redis==5.0.1
+Requires-Dist: Flask==3.0.0
 Requires-Dist: oic==1.6.1
-Requires-Dist: prometheus_client==0.17.1
-Requires-Dist: boto3==1.28.5
+Requires-Dist: prometheus_client==0.19.0
+Requires-Dist: boto3==1.29.5
 Provides-Extra: oracle
 Requires-Dist: cx_oracle==8.3.0; extra == "oracle"
 Provides-Extra: mongo
-Requires-Dist: pymongo==4.4.1; extra == "mongo"
+Requires-Dist: pymongo==4.6.0; extra == "mongo"
 Provides-Extra: postgresql
-Requires-Dist: psycopg2-binary==2.9.6; extra == "postgresql"
+Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
 Provides-Extra: mysql
 Requires-Dist: PyMySQL==1.1.0; extra == "mysql"
 Provides-Extra: kerberos
@@ -55,19 +55,19 @@ Requires-Dist: pykerberos~=1.2.4; extra == "kerberos"
 Requires-Dist: requests-kerberos>=0.14.0; extra == "kerberos"
 Provides-Extra: globus
 Requires-Dist: PyYAML==6.0.1; extra == "globus"
-Requires-Dist: globus-sdk==3.24.0; extra == "globus"
+Requires-Dist: globus-sdk==3.32.0; extra == "globus"
 Provides-Extra: saml
-Requires-Dist: python3-saml==1.15.0; extra == "saml"
+Requires-Dist: python3-saml==1.16.0; extra == "saml"
 Provides-Extra: dev
 Requires-Dist: PyYAML==6.0.1; extra == "dev"
-Requires-Dist: pytest==7.4.0; extra == "dev"
-Requires-Dist: pytest-xdist~=3.3.1; extra == "dev"
-Requires-Dist: pyflakes==3.0.1; extra == "dev"
-Requires-Dist: flake8==6.0.0; extra == "dev"
-Requires-Dist: pycodestyle==2.10.0; extra == "dev"
-Requires-Dist: pylint==3.0.0a7; extra == "dev"
-Requires-Dist: virtualenv==20.24.0; extra == "dev"
+Requires-Dist: pytest==7.4.3; extra == "dev"
+Requires-Dist: pytest-xdist~=3.5.0; extra == "dev"
+Requires-Dist: pyflakes==3.1.0; extra == "dev"
+Requires-Dist: flake8==6.1.0; extra == "dev"
+Requires-Dist: pycodestyle==2.11.0; extra == "dev"
+Requires-Dist: pylint==3.0.2; extra == "dev"
+Requires-Dist: virtualenv==20.24.7; extra == "dev"
 Requires-Dist: xmltodict==0.13.0; extra == "dev"
-Requires-Dist: pytz==2023.3; extra == "dev"
+Requires-Dist: pytz==2023.3.post1; extra == "dev"
 Requires-Dist: pydoc-markdown~=4.8.2; extra == "dev"
-Requires-Dist: sh~=2.0.4; extra == "dev"
+Requires-Dist: sh~=2.0.6; extra == "dev"

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/alembicrevision.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ALEMBIC_REVISION = '27e3a68927fb'  # the current alembic head revision
+ALEMBIC_REVISION = 'a08fa8de1545'  # the current alembic head revision

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/api/request.py

@@ -17,7 +17,7 @@
 Interface for the requests abstraction layer
 """
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from rucio.api import permission
 from rucio.common import exception
@@ -199,7 +199,6 @@ def list_requests(src_rses, dst_rses, states, issuer, vo='def', *, session: "Ses
 def list_requests_history(src_rses, dst_rses, states, issuer, vo='def', offset=None, limit=None, *, session: "Session"):
     """
     List all historical requests in a specific state from a source RSE to a destination RSE.
-
     :param src_rses: source RSEs.
     :param dst_rses: destination RSEs.
     :param states: list of request states.
@@ -218,3 +217,27 @@ def list_requests_history(src_rses, dst_rses, states, issuer, vo='def', offset=N
     for req in request.list_requests_history(src_rse_ids, dst_rse_ids, states, offset, limit, session=session):
         req = req.to_dict()
         yield api_update_return_dict(req, session=session)
+
+
+@read_session
+def get_request_metrics(src_rse: Optional[str], dst_rse: Optional[str], activity: Optional[str], issuer, vo='def', *, session: "Session"):
+    """
+    Get statistics of requests in a specific state grouped by source RSE, destination RSE, and activity.
+
+    :param src_rse: source RSE.
+    :param dst_rse: destination RSE.
+    :param activity: activity
+    :param issuer: Issuing account as a string.
+    :param session: The database session in use.
+    """
+    src_rse_id = None
+    if src_rse:
+        src_rse_id = get_rse_id(rse=src_rse, vo=vo, session=session)
+    dst_rse_id = None
+    if dst_rse:
+        dst_rse_id = get_rse_id(rse=dst_rse, vo=vo, session=session)
+    kwargs = {'issuer': issuer}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='get_request_metrics', kwargs=kwargs, session=session):
+        raise exception.AccessDenied(f'{issuer} cannot get request statistics')
+
+    return request.get_request_metrics(dest_rse_id=dst_rse_id, src_rse_id=src_rse_id, activity=activity, session=session)

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/client/didclient.py

@@ -661,19 +661,6 @@ class DIDClient(BaseClient):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def add_temporary_dids(self, dids):
-        """
-        Bulk add temporary data identifiers.
-
-        :param dids: A list of dids.
-        """
-        url = build_url(choice(self.list_hosts), path='tmp_dids')
-        r = self._send_request(url, type_='POST', data=dumps(dids))
-        if r.status_code == codes.created:
-            return True
-        exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
-        raise exc_cls(exc_msg)
-
     def list_archive_content(self, scope, name):
         """
         List archive contents.

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/core/did_meta_plugins/filter_engine.py

@@ -368,7 +368,7 @@ class FilterEngine:
                     # Infer what type key should be cast to from typecasting the value in the expression.
                     try:
                         if isinstance(value, int):                                          # this could be bool or int (as bool subclass of int)
-                            if type(value) == bool:
+                            if isinstance(value, bool):
                                 if is_in_json_column:
                                     expression = "({}->>'{}')::boolean {} {}".format(jsonb_column, key, POSTGRES_OP_MAP[oper], str(value).lower())
                                 else:
@@ -473,7 +473,7 @@ class FilterEngine:
                             # Infer what type key should be cast to from typecasting the value in the expression.
                             try:
                                 if isinstance(value, int):                                          # this could be bool or int (as bool subclass of int)
-                                    if type(value) == bool:
+                                    if isinstance(value, bool):
                                         expression = oper(json_column[key].as_boolean(), value)
                                     else:
                                         expression = oper(json_column[key].as_float(), value)       # cast as float, not integer, to avoid potentially losing precision in key

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/core/oidc.py

@@ -13,15 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import json
 import logging
 import random
 import subprocess
 import traceback
 from datetime import datetime, timedelta
-from typing import Any, TYPE_CHECKING, Optional
-from urllib.parse import urlparse, parse_qs
+from typing import Any, Final, TYPE_CHECKING, Optional
+from urllib.parse import urljoin, urlparse, parse_qs
 
+import requests
+from dogpile.cache.api import NoValue
 from jwkest.jws import JWS
 from jwkest.jwt import JWT
 from math import floor
@@ -36,6 +39,7 @@ from sqlalchemy import delete, select, update
 from sqlalchemy.sql.expression import true
 
 from rucio.common import types
+from rucio.common.cache import make_region_memcached
 from rucio.common.config import config_get, config_get_int
 from rucio.common.exception import (CannotAuthenticate, CannotAuthorize,
                                     RucioException)
@@ -52,7 +56,12 @@ from rucio.db.sqla.session import read_session, transactional_session
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
+# The WLCG Common JWT Profile dictates that the lifetime of access and ID tokens
+# should range from five minutes to six hours.
+TOKEN_MIN_LIFETIME: Final = int(timedelta(minutes=5).total_seconds())
+TOKEN_MAX_LIFETIME: Final = int(timedelta(hours=6).total_seconds())
 
+REGION: Final = make_region_memcached(expiration_time=TOKEN_MAX_LIFETIME)
 METRICS = MetricManager(module=__name__)
 
 # worokaround for a bug in pyoidc (as of Dec 2019)
@@ -75,6 +84,78 @@ LEEWAY_SECS = 120
 # --> check 'profile' info (requested profile scope)
 
 
+@METRICS.time_it
+def _token_cache_get(
+    key: str,
+    min_lifetime: int = TOKEN_MIN_LIFETIME,
+) -> Optional[str]:
+    """Retrieve a token from the cache.
+
+    Return ``None`` if the cache backend did not return a value, the value is
+    not a valid JWT, or the token has a remaining lifetime less than
+    ``min_lifetime`` seconds.
+    """
+    value = REGION.get(key)
+    if isinstance(value, NoValue):
+        METRICS.counter('token_cache.miss').inc()
+        return None
+
+    try:
+        assert isinstance(value, str)
+        payload = JWT().unpack(value).payload()
+    except Exception:
+        METRICS.counter('token_cache.invalid').inc()
+        return None
+
+    now = datetime.utcnow().timestamp()
+    expiration = payload.get('exp', 0)    # type: ignore
+    if now + min_lifetime > expiration:
+        METRICS.counter('token_cache.expired').inc()
+        return None
+
+    METRICS.counter('token_cache.hit').inc()
+    return value
+
+
+def _token_cache_set(key: str, value: str) -> None:
+    """Store a token in the cache."""
+    REGION.set(key, value)
+
+
+def request_token(audience: str, scope: str, use_cache: bool = True) -> Optional[str]:
+    """Request a token from the provider.
+
+    Return ``None`` if the configuration was not loaded properly or the request
+    was unsuccessful.
+    """
+    if not all([OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_PROVIDER_ENDPOINT]):
+        if OIDC_CONFIGURATION_RUN or not __load_oidc_configuration():
+            return None
+
+    key = hashlib.md5(f'audience={audience};scope={scope}'.encode()).hexdigest()
+
+    if use_cache and (token := _token_cache_get(key)):
+        return token
+
+    try:
+        response = requests.post(url=OIDC_PROVIDER_ENDPOINT,
+                                 auth=(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET),
+                                 data={'grant_type': 'client_credentials',
+                                       'audience': audience,
+                                       'scope': scope})
+        response.raise_for_status()
+        payload = response.json()
+        token = payload['access_token']
+    except Exception:
+        logging.debug('Failed to procure a token', exc_info=True)
+        return None
+
+    if use_cache:
+        _token_cache_set(key, token)
+
+    return token
+
+
 def __get_rucio_oidc_clients(keytimeout: int = 43200) -> tuple[dict, dict]:
     """
     Creates a Rucio OIDC Client instances per Identity Provider (IdP)
@@ -125,6 +206,11 @@ def __get_rucio_oidc_clients(keytimeout: int = 43200) -> tuple[dict, dict]:
 # global variables to represent the IdP clients
 OIDC_CLIENTS = {}
 OIDC_ADMIN_CLIENTS = {}
+# New-style token support.
+OIDC_CLIENT_ID = ''
+OIDC_CLIENT_SECRET = ''
+OIDC_PROVIDER_ENDPOINT = ''
+OIDC_CONFIGURATION_RUN = False
 
 
 def __initialize_oidc_clients() -> None:
@@ -143,8 +229,31 @@ def __initialize_oidc_clients() -> None:
         pass
 
 
-# try loading OIDC clients uppon module import
-__initialize_oidc_clients()
+def __load_oidc_configuration() -> bool:
+    """Load the configuration for the new-style token support."""
+    global OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_PROVIDER_ENDPOINT, OIDC_CONFIGURATION_RUN
+
+    OIDC_CONFIGURATION_RUN = True
+
+    if not IDPSECRETS:
+        logging.error('Configuration option "idpsecrets" in section "oidc" is not set')
+        return False
+    if not ADMIN_ISSUER_ID:
+        logging.error('Configuration option "admin_issuer" in section "oidc" is not set')
+        return False
+
+    try:
+        with open(IDPSECRETS) as f:
+            data = json.load(f)
+            OIDC_CLIENT_ID = data[ADMIN_ISSUER_ID]['client_id']
+            OIDC_CLIENT_SECRET = data[ADMIN_ISSUER_ID]['client_secret']
+            OIDC_PROVIDER_ENDPOINT = urljoin(data[ADMIN_ISSUER_ID]['issuer'], 'token')
+    except Exception:
+        logging.error('Failed to parse configuration file "%s"', IDPSECRETS,
+                      exc_info=True)
+        return False
+    else:
+        return True
 
 
 def __get_init_oidc_client(token_object: models.Token = None, token_type: str = None, **kwargs) -> dict[Any, Any]:

{rucio-32.6.0.post1 → rucio-33.0.0rc2}/lib/rucio/core/permission/atlas.py

@@ -87,6 +87,7 @@ def has_permission(issuer, action, kwargs, *, session: "Optional[Session]" = Non
             'set_rse_usage': perm_set_rse_usage,
             'set_rse_limits': perm_set_rse_limits,
             'get_request_by_did': perm_get_request_by_did,
+            'get_request_metrics': perm_get_request_metrics,
             'cancel_request': perm_cancel_request,
             'get_next': perm_get_next,
             'set_local_account_limit': perm_set_local_account_limit,
@@ -940,6 +941,18 @@ def perm_get_request_by_did(issuer, kwargs, *, session: "Optional[Session]" = No
     return True
 
 
+def perm_get_request_metrics(issuer, kwargs, *, session: "Optional[Session]" = None):
+    """
+    Checks if an account can get the request stats
+
+    :param issuer: Account identifier which issues the command.
+    :param kwargs: List of arguments for the action.
+    :param session: The DB session to use
+    :returns: True if account is allowed, otherwise False
+    """
+    return _is_root(issuer) or has_account_attribute(account=issuer, key='admin', session=session)
+
+
 def perm_cancel_request(issuer, kwargs, *, session: "Optional[Session]" = None):
     """
     Checks if an account can cancel a request.

rucio-33.0.0rc2/lib/rucio/core/plugins.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import os
+from configparser import NoOptionError, NoSectionError
+from typing import Any, Callable, Dict, TypeVar, Type
+
+from rucio.common import config
+from rucio.common.exception import InvalidAlgorithmName
+from rucio.common.utils import check_policy_package_version
+from rucio.core.vo import list_vos
+
+
+PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
+
+
+class PolicyPackageAlgorithms():
+    """
+    Base class for Rucio Policy Package Algorithms
+
+    ALGORITHMS is of type Dict[str, Dict[str. Callable[..., Any]]]
+    where the key is the algorithm type and the value is a dictionary of algorithm names and their callables
+    """
+    _ALGORITHMS: Dict[str, Dict[str, Callable[..., Any]]] = {}
+    _loaded_policy_modules = False
+
+    def __init__(self) -> None:
+        if not self._loaded_policy_modules:
+            self._register_all_policy_package_algorithms()
+            self._loaded_policy_modules = True
+
+    @classmethod
+    def _get_one_algorithm(cls: Type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> Callable[..., Any]:
+        """
+        Get the algorithm from the dictionary of algorithms
+        """
+        return cls._ALGORITHMS[algorithm_type][name]
+
+    @classmethod
+    def _get_algorithms(cls: Type[PolicyPackageAlgorithmsT], algorithm_type: str) -> Dict[str, Callable[..., Any]]:
+        """
+        Get the dictionary of algorithms for a given type
+        """
+        return cls._ALGORITHMS[algorithm_type]
+
+    @classmethod
+    def _register(
+            cls: Type[PolicyPackageAlgorithmsT],
+            algorithm_type: str, algorithm_dict: Dict[str, Callable[..., Any]]) -> None:
+        """
+        Provided a dictionary of callable function,
+        and the associated algorithm type,
+        register it as one of the valid algorithms.
+        """
+        if algorithm_type in cls._ALGORITHMS:
+            cls._ALGORITHMS[algorithm_type].update(algorithm_dict)
+        else:
+            cls._ALGORITHMS[algorithm_type] = algorithm_dict
+
+    @classmethod
+    def _supports(cls: Type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> bool:
+        """
+        Check if a algorithm is supported by the plugin
+        """
+        return name in cls._ALGORITHMS.get(algorithm_type, {})
+
+    @classmethod
+    def _register_all_policy_package_algorithms(cls: Type[PolicyPackageAlgorithmsT]) -> None:
+        '''
+        Loads all the algorithms of a given type from the policy package(s) and registers them
+        :param algorithm_type: the type of algorithm to register (e.g. 'surl', 'lfn2pfn')
+        :param dictionary: the dictionary to register them in
+        :param vo: the name of the relevant VO (None for single VO)
+        '''
+        try:
+            multivo = config.config_get_bool('common', 'multi_vo')
+        except (NoOptionError, NoSectionError):
+            multivo = False
+        if not multivo:
+            # single policy package
+            cls._try_importing_policy()
+        else:
+            # determine whether on client or server
+            client = False
+            if 'RUCIO_CLIENT_MODE' not in os.environ:
+                if not config.config_has_section('database') and config.config_has_section('client'):
+                    client = True
+            else:
+                if os.environ['RUCIO_CLIENT_MODE']:
+                    client = True
+
+            # on client, only register algorithms for selected VO
+            if client:
+                if 'RUCIO_VO' in os.environ:
+                    vo = os.environ['RUCIO_VO']
+                else:
+                    try:
+                        vo = str(config.config_get('client', 'vo'))
+                    except (NoOptionError, NoSectionError):
+                        vo = 'def'
+                cls._try_importing_policy(vo)
+            # on server, list all VOs and register their algorithms
+            else:
+                # policy package per VO
+                vos = list_vos()
+                for vo in vos:
+                    cls._try_importing_policy(vo['vo'])
+
+    @classmethod
+    def _try_importing_policy(cls: Type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
+        try:
+            env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
+            package = getattr(os.environ, env_name, "")
+            if not package:
+                package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
+
+            check_policy_package_version(package)
+            module = importlib.import_module(package)
+
+            if hasattr(module, 'get_algorithms'):
+                all_algorithms = module.get_algorithms()
+
+                # check that the names are correctly prefixed for multi-VO
+                if vo:
+                    for _, algorithms in all_algorithms.items():
+                        for k in algorithms.keys():
+                            if not k.lower().startswith(vo.lower()):
+                                raise InvalidAlgorithmName(k, vo)
+
+                # Updates the dictionary with the algorithms from the policy package
+                for algorithm_type, algorithm_dict in all_algorithms.items():
+                    cls._register(algorithm_type, algorithm_dict)
+
+        except (NoOptionError, NoSectionError, ImportError):
+            pass