rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/core/naming_convention.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,27 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from re import match, compile, error
+from re import compile, error, match
 from traceback import format_exc
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 from dogpile.cache.api import NO_VALUE
+from sqlalchemy import and_, delete, select
 from sqlalchemy.exc import IntegrityError
 
 from rucio.common.cache import make_region_memcached
-from rucio.common.exception import Duplicate, RucioException, InvalidObject
+from rucio.common.exception import Duplicate, InvalidObject, RucioException
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import KeyType
 from rucio.db.sqla.session import read_session, transactional_session
 
 if TYPE_CHECKING:
+    from typing import TypedDict
+
     from sqlalchemy.orm import Session
 
+    from rucio.common.types import InternalScope
+
+    class NamingConventionDict(TypedDict):
+        scope: InternalScope
+        regexp: str
+
 REGION = make_region_memcached(expiration_time=900)
 
 
 @transactional_session
-def add_naming_convention(scope, regexp, convention_type, *, session: "Session"):
+def add_naming_convention(
+    scope: "InternalScope",
+    regexp: str,
+    convention_type: KeyType,
+    *,
+    session: "Session"
+) -> None:
     """
     add a naming convention for a given scope
 
@@ -60,7 +74,12 @@ def add_naming_convention(scope, regexp, convention_type, *, session: "Session")
 
 
 @read_session
-def get_naming_convention(scope, convention_type, *, session: "Session"):
+def get_naming_convention(
+    scope: "InternalScope",
+    convention_type: KeyType,
+    *,
+    session: "Session"
+) -> Optional[str]:
     """
     Get the naming convention for a given scope
 
@@ -70,15 +89,22 @@ def get_naming_convention(scope, convention_type, *, session: "Session"):
 
     :returns: the regular expression.
     """
-    query = session.query(models.NamingConvention.regexp).\
-        filter(models.NamingConvention.scope == scope).\
-        filter(models.NamingConvention.convention_type == convention_type)
-    for row in query:
-        return row[0]
+    stmt = select(
+        models.NamingConvention.regexp
+    ).where(
+        and_(models.NamingConvention.scope == scope,
+             models.NamingConvention.convention_type == convention_type)
+    )
+    return session.execute(stmt).scalar()
 
 
 @transactional_session
-def delete_naming_convention(scope, convention_type, *, session: "Session"):
+def delete_naming_convention(
+    scope: "InternalScope",
+    convention_type: KeyType,
+    *,
+    session: "Session"
+) -> int:
     """
     delete a naming convention for a given scope
 
@@ -87,14 +113,19 @@ def delete_naming_convention(scope, convention_type, *, session: "Session"):
     :param convention_type: the did_type on which the regexp should apply.
     :param session: The database session in use.
     """
-    REGION.delete(scope.internal)
-    return session.query(models.NamingConvention) \
-        .filter_by(scope=scope, convention_type=convention_type) \
-        .delete()
+    if scope.internal is not None:
+        REGION.delete(scope.internal)
+    stmt = delete(
+        models.NamingConvention
+    ).where(
+        and_(models.NamingConvention.scope == scope,
+             models.NamingConvention.convention_type == convention_type)
+    )
+    return session.execute(stmt).rowcount
 
 
 @read_session
-def list_naming_conventions(*, session: "Session"):
+def list_naming_conventions(*, session: "Session") -> list["NamingConventionDict"]:
     """
     List all naming conventions.
 
@@ -102,13 +133,21 @@ def list_naming_conventions(*, session: "Session"):
 
     :returns: a list of dictionaries.
     """
-    query = session.query(models.NamingConvention.scope,
-                          models.NamingConvention.regexp)
-    return [row._asdict() for row in query]
+    stmt = select(
+        models.NamingConvention.scope,
+        models.NamingConvention.regexp
+    )
+    return [cast("NamingConventionDict", row._asdict()) for row in session.execute(stmt).all()]
 
 
 @read_session
-def validate_name(scope, name, did_type, *, session: "Session"):
+def validate_name(
+    scope: "InternalScope",
+    name: str,
+    did_type: str,
+    *,
+    session: "Session"
+) -> Optional[dict[str, Any]]:
     """
     Validate a name according to a naming convention.
 
@@ -120,10 +159,11 @@ def validate_name(scope, name, did_type, *, session: "Session"):
 
     :returns: a dictionary with metadata.
     """
-    if scope.external.startswith('user'):
-        return {'project': 'user'}
-    elif scope.external.startswith('group'):
-        return {'project': 'group'}
+    if scope.external is not None:
+        if scope.external.startswith('user'):
+            return {'project': 'user'}
+        elif scope.external.startswith('group'):
+            return {'project': 'group'}
 
     # Check if naming convention can be found in cache region
     regexp = REGION.get(scope.internal)
@@ -131,13 +171,14 @@ def validate_name(scope, name, did_type, *, session: "Session"):
         regexp = get_naming_convention(scope=scope,
                                        convention_type=KeyType.DATASET,
                                        session=session)
-        regexp and REGION.set(scope.internal, regexp)
+        if scope.internal is not None:
+            regexp and REGION.set(scope.internal, regexp)
 
     if not regexp:
         return
 
     # Validate with regexp
-    groups = match(regexp, str(name))
+    groups = match(regexp, str(name))  # type: ignore
     if groups:
         meta = groups.groupdict()
         # Hack to get task_id from version

rucio/core/nongrid_trace.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,6 +16,7 @@ import json
 import logging.handlers
 import random
 import socket
+from typing import TYPE_CHECKING, Any, Union, overload
 
 import stomp
 
@@ -24,6 +24,9 @@ from rucio.common.config import config_get, config_get_int
 from rucio.common.logging import rucio_log_formatter
 from rucio.core.monitor import MetricManager
 
+if TYPE_CHECKING:
+    from datetime import datetime
+
 METRICS = MetricManager(module=__name__)
 
 CONFIG_COMMON_LOGLEVEL = getattr(logging, config_get('common', 'loglevel', raise_exception=False, default='DEBUG').upper())
@@ -80,14 +83,23 @@ for broker in BROKERS_RESOLVED:
     CONNS.append(stomp.Connection(host_and_ports=[(broker, PORT)], vhost=VHOST, reconnect_attempts_max=3))
 
 
-def date_handler(obj):
-    '''
-    '''
+@overload
+def date_handler(obj: "datetime") -> str:
+    ...
+
+
+@overload
+def date_handler(obj: object) -> object:
+    ...
+
+
+def date_handler(obj: Any) -> Union[str, object]:
+    """ Format dates to ISO format. """
     return obj.isoformat() if hasattr(obj, 'isoformat') else obj
 
 
 @METRICS.count_it
-def trace(payload):
+def trace(payload: dict[str, Any]) -> None:
     """
     Write a trace to the buffer log file and send it to active mq.
 

rucio/core/oidc.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,46 +12,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import json
 import logging
-import random
 import subprocess
 import traceback
 from datetime import datetime, timedelta
-from typing import Any, TYPE_CHECKING, Optional
-from urllib.parse import urlparse, parse_qs
+from math import floor
+from secrets import choice
+from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from urllib.parse import parse_qs, urljoin, urlparse
 
+import requests
+from dogpile.cache.api import NoValue
 from jwkest.jws import JWS
 from jwkest.jwt import JWT
-from math import floor
 from oic import rndstr
 from oic.oauth2.message import CCAccessTokenRequest
-from oic.oic import Client, Grant, Token, REQUEST2ENDPOINT
-from oic.oic.message import (AccessTokenResponse, AuthorizationResponse,
-                             Message, RegistrationResponse)
+from oic.oic import REQUEST2ENDPOINT, Client, Grant, Token
+from oic.oic.message import AccessTokenResponse, AuthorizationResponse, Message, RegistrationResponse
 from oic.utils import time_util
 from oic.utils.authn.client import CLIENT_AUTHN_METHOD
 from sqlalchemy import delete, select, update
 from sqlalchemy.sql.expression import true
 
 from rucio.common import types
+from rucio.common.cache import make_region_memcached
 from rucio.common.config import config_get, config_get_int
-from rucio.common.exception import (CannotAuthenticate, CannotAuthorize,
-                                    RucioException)
+from rucio.common.exception import CannotAuthenticate, CannotAuthorize, RucioException
 from rucio.common.stopwatch import Stopwatch
 from rucio.common.utils import all_oidc_req_claims_present, build_url, val_to_space_sep_str
 from rucio.core.account import account_exists
 from rucio.core.identity import exist_identity_account, get_default_account
 from rucio.core.monitor import MetricManager
-from rucio.db.sqla import filter_thread_work
-from rucio.db.sqla import models
+from rucio.db.sqla import filter_thread_work, models
 from rucio.db.sqla.constants import IdentityType
 from rucio.db.sqla.session import read_session, transactional_session
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
+# The WLCG Common JWT Profile dictates that the lifetime of access and ID tokens
+# should range from five minutes to six hours.
+TOKEN_MIN_LIFETIME: Final = config_get_int('oidc', 'token_min_lifetime', default=300)
+TOKEN_MAX_LIFETIME: Final = config_get_int('oidc', 'token_max_lifetime', default=21600)
 
+REGION: Final = make_region_memcached(expiration_time=TOKEN_MAX_LIFETIME)
 METRICS = MetricManager(module=__name__)
 
 # worokaround for a bug in pyoidc (as of Dec 2019)
@@ -75,6 +80,81 @@ LEEWAY_SECS = 120
 # --> check 'profile' info (requested profile scope)
 
 
+@METRICS.time_it
+def _token_cache_get(
+    key: str,
+    min_lifetime: int = TOKEN_MIN_LIFETIME,
+) -> Optional[str]:
+    """Retrieve a token from the cache.
+
+    Return ``None`` if the cache backend did not return a value, the value is
+    not a valid JWT, or the token has a remaining lifetime less than
+    ``min_lifetime`` seconds.
+    """
+    value = REGION.get(key)
+    if isinstance(value, NoValue):
+        METRICS.counter('token_cache.miss').inc()
+        return None
+
+    if isinstance(value, str):
+        try:
+            payload = JWT().unpack(value).payload()
+        except Exception:
+            METRICS.counter('token_cache.invalid').inc()
+            return None
+    else:
+        METRICS.counter('token_cache.invalid').inc()
+        return None
+
+    now = datetime.utcnow().timestamp()
+    expiration = payload.get('exp', 0)    # type: ignore
+    if now + min_lifetime > expiration:
+        METRICS.counter('token_cache.expired').inc()
+        return None
+
+    METRICS.counter('token_cache.hit').inc()
+    return value
+
+
+def _token_cache_set(key: str, value: str) -> None:
+    """Store a token in the cache."""
+    REGION.set(key, value)
+
+
+def request_token(audience: str, scope: str, use_cache: bool = True) -> Optional[str]:
+    """Request a token from the provider.
+
+    Return ``None`` if the configuration was not loaded properly or the request
+    was unsuccessful.
+    """
+    if not all([OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_PROVIDER_ENDPOINT]):
+        if OIDC_CONFIGURATION_RUN or not __load_oidc_configuration():
+            return None
+
+    key = hashlib.md5(f'audience={audience};scope={scope}'.encode()).hexdigest()
+
+    if use_cache and (token := _token_cache_get(key)):
+        return token
+
+    try:
+        response = requests.post(url=OIDC_PROVIDER_ENDPOINT,
+                                 auth=(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET),
+                                 data={'grant_type': 'client_credentials',
+                                       'audience': audience,
+                                       'scope': scope})
+        response.raise_for_status()
+        payload = response.json()
+        token = payload['access_token']
+    except Exception:
+        logging.debug('Failed to procure a token', exc_info=True)
+        return None
+
+    if use_cache:
+        _token_cache_set(key, token)
+
+    return token
+
+
 def __get_rucio_oidc_clients(keytimeout: int = 43200) -> tuple[dict, dict]:
     """
     Creates a Rucio OIDC Client instances per Identity Provider (IdP)
@@ -125,6 +205,11 @@ def __get_rucio_oidc_clients(keytimeout: int = 43200) -> tuple[dict, dict]:
 # global variables to represent the IdP clients
 OIDC_CLIENTS = {}
 OIDC_ADMIN_CLIENTS = {}
+# New-style token support.
+OIDC_CLIENT_ID = ''
+OIDC_CLIENT_SECRET = ''
+OIDC_PROVIDER_ENDPOINT = ''
+OIDC_CONFIGURATION_RUN = False
 
 
 def __initialize_oidc_clients() -> None:
@@ -143,8 +228,40 @@ def __initialize_oidc_clients() -> None:
         pass
 
 
-# try loading OIDC clients uppon module import
-__initialize_oidc_clients()
+def __load_oidc_configuration() -> bool:
+    """Load the configuration for the new-style token support."""
+    global OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_PROVIDER_ENDPOINT, OIDC_CONFIGURATION_RUN
+
+    OIDC_CONFIGURATION_RUN = True
+
+    if not IDPSECRETS:
+        logging.error('Configuration option "idpsecrets" in section "oidc" is not set')
+        return False
+    if not ADMIN_ISSUER_ID:
+        logging.error('Configuration option "admin_issuer" in section "oidc" is not set')
+        return False
+
+    try:
+        with open(IDPSECRETS) as f:
+            data = json.load(f)
+            OIDC_CLIENT_ID = data[ADMIN_ISSUER_ID]['client_id']
+            OIDC_CLIENT_SECRET = data[ADMIN_ISSUER_ID]['client_secret']
+            issuer = data[ADMIN_ISSUER_ID]['issuer']
+    except Exception:
+        logging.error('Failed to parse configuration file "%s"', IDPSECRETS,
+                      exc_info=True)
+        return False
+    try:
+        oidc_discover_url = urljoin(issuer, '.well-known/openid-configuration')
+        response = requests.get(oidc_discover_url)
+        response.raise_for_status()
+        payload = response.json()
+        OIDC_PROVIDER_ENDPOINT = payload['token_endpoint']
+    except (requests.HTTPError, requests.JSONDecodeError, KeyError):
+        logging.error('Failed to discover token endpoint', exc_info=True)
+        return False
+
+    return True
 
 
 def __get_init_oidc_client(token_object: models.Token = None, token_type: str = None, **kwargs) -> dict[Any, Any]:
@@ -189,11 +306,11 @@ def __get_init_oidc_client(token_object: models.Token = None, token_type: str =
             token = ''
             if not token_type:
                 token_type = kwargs.get('token_type', None)
-            if token_type == 'subject_token':
+            if token_type == 'subject_token':  # noqa: S105
                 token = token_object.token
                 # do not remove - even though None, oic expects this key to exist
                 auth_args["redirect_uri"] = None
-            if token_type == 'refresh_token':
+            if token_type == 'refresh_token':  # noqa: S105
                 token = token_object.refresh_token
                 # do not remove - even though None, oic expects this key to exist
                 auth_args["redirect_uri"] = None
@@ -219,7 +336,7 @@ def __get_init_oidc_client(token_object: models.Token = None, token_type: str =
             if not redirect_url:
                 redirect_to = kwargs.get("redirect_to", "auth/oidc_token")
                 redirect_urls = [u for u in client_secret["redirect_uris"] if redirect_to in u]
-                redirect_url = random.choice(redirect_urls)
+                redirect_url = choice(redirect_urls)
             if not redirect_url:
                 raise CannotAuthenticate("Could not pick any redirect URL(s) from the ones defined "
                                          + "in Rucio OIDC Client configuration file.")  # NOQA: W503
@@ -275,7 +392,7 @@ def get_auth_oidc(account: str, *, session: "Session", **kwargs) -> str:
               OR a redirection url to be used in user's browser for authentication.
     """
     # TO-DO - implement a check if that account already has a valid
-    # token withthe required scope and audience and return such token !
+    # token with the required scope and audience and return such token !
     auth_scope = kwargs.get('auth_scope', EXPECTED_OIDC_SCOPE)
     if not auth_scope:
         auth_scope = EXPECTED_OIDC_SCOPE
@@ -360,7 +477,12 @@ def get_auth_oidc(account: str, *, session: "Session", **kwargs) -> str:
 
 
 @transactional_session
-def get_token_oidc(auth_query_string: str, ip: str = None, *, session: "Session"):
+def get_token_oidc(
+    auth_query_string: str,
+    ip: Optional[str] = None,
+    *,
+    session: "Session"
+) -> Optional[dict[str, Optional[Union[str, bool]]]]:
     """
     After Rucio User got redirected to Rucio /auth/oidc_token (or /auth/oidc_code)
     REST endpoints with authz code and session state encoded within the URL.
@@ -651,8 +773,8 @@ def get_token_for_account_operation(account: str, req_audience: str = None, req_
         # supported by Rucio server (have OIDC admin client registered as well)
         # that is why we take the issuer of the account identity that has an active/valid token
         # and look for admin account identity which has this issuer assigned
-        # requestor should always have at least one active subject token unless it is root
-        # this is why we first discover if the requestor is root or not
+        # requester should always have at least one active subject token unless it is root
+        # this is why we first discover if the requester is root or not
         get_token_for_adminacc = False
         admin_identity = None
         admin_issuer = None
@@ -662,10 +784,10 @@ def get_token_for_account_operation(account: str, req_audience: str = None, req_
         preferred_issuer = None
         for token in account_tokens:
             preferred_issuer = token.identity.split(", ")[1].split("=")[1]
-        # loop through all OIDC identities registerd for the account of the requestor
+        # loop through all OIDC identities registered for the account of the requester
         for identity in identities:
             issuer = identity.split(", ")[1].split("=")[1]
-            # compare the account of the requestor with the account of the admin
+            # compare the account of the requester with the account of the admin
             if account == admin_iss_acc_idt_dict[issuer][0]:
                 # take first matching case which means root is requesting OIDC authentication
                 admin_identity = admin_iss_acc_idt_dict[issuer][1]
@@ -765,7 +887,7 @@ def get_token_for_account_operation(account: str, req_audience: str = None, req_
                     subject_token = token
             # if not proceed with token exchange
             if not subject_token:
-                subject_token = random.choice(account_tokens)
+                subject_token = choice(account_tokens)
             exchanged_token = __exchange_token_oidc(subject_token,
                                                     scope=req_scope,
                                                     audience=req_audience,
@@ -813,7 +935,7 @@ def __exchange_token_oidc(subject_token_object: models.Token, *, session: "Sessi
     if not grant_type:
         grant_type = EXCHANGE_GRANT_TYPE
     try:
-        oidc_dict = __get_init_oidc_client(token_object=subject_token_object, token_type="subject_token")
+        oidc_dict = __get_init_oidc_client(token_object=subject_token_object, token_type="subject_token")  # noqa: S106
         oidc_client = oidc_dict['client']
         args = {"subject_token": subject_token_object.token,
                 "scope": jwt_row_dict['authz_scope'],
@@ -891,7 +1013,7 @@ def __change_refresh_state(token: str, refresh: bool = False, *, session: "Sessi
 
 
 @transactional_session
-def refresh_cli_auth_token(token_string: str, account: str, *, session: "Session"):
+def refresh_cli_auth_token(token_string: str, account: str, *, session: "Session") -> Optional[tuple[str, int]]:
     """
     Checks if there is active refresh token and if so returns
     either active token with expiration timestamp or requests a new
@@ -1057,7 +1179,7 @@ def __refresh_token_oidc(token_object: models.Token, *, session: "Session"):
         if datetime.utcnow() - extra_dict['refresh_start'] > timedelta(hours=extra_dict['refresh_lifetime']):
             __change_refresh_state(token_object.token, refresh=False, session=session)
             return None
-        oidc_dict = __get_init_oidc_client(token_object=token_object, token_type="refresh_token")
+        oidc_dict = __get_init_oidc_client(token_object=token_object, token_type="refresh_token")  # noqa: S106
         oidc_client = oidc_dict['client']
         # getting a new refreshed set of tokens
         state = oidc_dict['state']
@@ -1169,7 +1291,7 @@ def __get_keyvalues_from_claims(token: str, keys=None):
         for key in keys:
             value = ''
             if key in claims:
-                value = val_to_space_sep_str(claims[key])
+                value = val_to_space_sep_str(claims[key])  # type: ignore
             resdict[key] = value
         return resdict
     except Exception as error:
@@ -1293,7 +1415,7 @@ def validate_jwt(json_web_token: str, *, session: "Session") -> dict[str, Any]:
         # try to get it from IdP introspection endpoint
         # TO-BE-REMOVED - once all IdPs support scope and audience in token claims !!!
         if not token_dict['authz_scope'] or not token_dict['audience']:
-            clprocess = subprocess.Popen(['curl', '-s', '-L', '-u', '%s:%s'
+            clprocess = subprocess.Popen(['curl', '-s', '-L', '-u', '%s:%s'  # noqa: S607
                                           % (oidc_client.client_id, oidc_client.client_secret),
                                           '-d', 'token=%s' % (json_web_token),
                                           oidc_client.introspection_endpoint],

rucio/core/permission/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,15 +15,18 @@
 import importlib
 from configparser import NoOptionError, NoSectionError
 from os import environ
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from rucio.common import config, exception
 from rucio.common.utils import check_policy_package_version
 
 if TYPE_CHECKING:
     from typing import Optional
+
     from sqlalchemy.orm import Session
 
+    from rucio.common.types import InternalAccount
+
 # dictionary of permission modules for each VO
 permission_modules = {}
 
@@ -66,13 +68,15 @@ if not multivo:
 
     try:
         module = importlib.import_module(POLICY)
+    except ModuleNotFoundError:
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     permission_modules["def"] = module
 
 
-def load_permission_for_vo(vo):
+def load_permission_for_vo(vo: str) -> None:
     GENERIC_FALLBACK = 'generic_multi_vo'
     if config.config_has_section('policy'):
         try:
@@ -95,13 +99,21 @@ def load_permission_for_vo(vo):
 
     try:
         module = importlib.import_module(POLICY)
+    except ModuleNotFoundError:
+        raise exception.PolicyPackageNotFound(POLICY)
     except ImportError:
-        raise exception.PolicyPackageNotFound('Module ' + POLICY + ' not found')
+        raise exception.ErrorLoadingPolicyPackage(POLICY)
 
     permission_modules[vo] = module
 
 
-def has_permission(issuer, action, kwargs, *, session: "Optional[Session]" = None):
+def has_permission(
+        issuer: "InternalAccount",
+        action: str,
+        kwargs: dict[str, Any],
+        *,
+        session: "Optional[Session]" = None
+) -> bool:
     if issuer.vo not in permission_modules:
         load_permission_for_vo(issuer.vo)
     return permission_modules[issuer.vo].has_permission(issuer, action, kwargs, session=session)