rucio 37.1.0.post1__py3-none-any.whl → 37.3.0__py3-none-any.whl

rucio/common/plugins.py

@@ -156,7 +156,7 @@ class PolicyPackageAlgorithms:
             # import from utils here to avoid circular import
 
             env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
-            package = getattr(os.environ, env_name, "")
+            package = os.getenv(env_name, "")
             if not package:
                 package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
 

rucio/common/utils.py

@@ -595,20 +595,17 @@ class ScopeExtractionAlgorithms(PolicyPackageAlgorithms):
     @staticmethod
     def extract_scope_default(did: str, scopes: Optional['Sequence[str]']) -> 'Sequence[str]':
         """
-        Default fallback scope extraction algorithm, based on the ATLAS scope extraction algorithm.
+        Default scope extraction algorithm. Extracts the scope from the DID.
 
         :param did: The DID to extract the scope from.
+        :param scopes: Not used in the default algorithm.
 
-        :returns: A tuple containing the extracted scope and the DID.
+        :returns: A tuple containing the extracted scope and the name.
         """
-        if did.find(':') > -1:
-            if len(did.split(':')) > 2:
-                raise RucioException('Too many colons. Cannot extract scope and name')
-            scope, name = did.split(':')[0], did.split(':')[1]
-            if name.endswith('/'):
-                name = name[:-1]
-            return scope, name
-        else:
+
+        # This block is ATLAS specific, to be removed in the future.
+        # More info at https://github.com/rucio/rucio/pull/7521
+        if did.find(':') == -1:
             scope = did.split('.')[0]
             if did.startswith('user') or did.startswith('group'):
                 scope = ".".join(did.split('.')[0:2])
@@ -616,6 +613,16 @@ class ScopeExtractionAlgorithms(PolicyPackageAlgorithms):
                 did = did[:-1]
             return scope, did
 
+        parts = did.split(':')
+        if len(parts) != 2:
+            msg = f"Cannot extract scope and name from DID {did}. The DID should have exactly one colon but found {len(parts)} colons."
+            raise RucioException(msg)
+        scope, name = parts
+        if not scope or not name:
+            msg = f"Cannot extract scope and name from DID {did}. Found empty scope or name."
+            raise RucioException(msg)
+        return scope, name
+
     @staticmethod
     def extract_scope_dirac(did: str, scopes: Optional['Sequence[str]']) -> 'Sequence[str]':
         # Default dirac scope extract algorithm. Scope is the second element in the LFN or the first one (VO name)

rucio/core/did.py

@@ -15,7 +15,6 @@
 import logging
 import random
 from datetime import datetime, timedelta
-from enum import Enum
 from hashlib import md5
 from re import match
 from typing import TYPE_CHECKING, Any, Literal, Optional, Union
@@ -254,6 +253,7 @@ def attach_dids(
         name: str,
         dids: "Sequence[Mapping[str, Any]]",
         account: "InternalAccount",
+        ignore_duplicate: bool = False,
         rse_id: Optional[str] = None,
         *,
         session: "Session",
@@ -265,10 +265,11 @@ def attach_dids(
     :param name: The data identifier name.
     :param dids: The content.
     :param account: The account owner.
+    :param ignore_duplicate: If True, ignore duplicate entries.
     :param rse_id: The RSE id for the replicas.
     :param session: The database session in use.
     """
-    return attach_dids_to_dids(attachments=[{'scope': scope, 'name': name, 'dids': dids, 'rse_id': rse_id}], account=account, session=session)
+    return attach_dids_to_dids(attachments=[{'scope': scope, 'name': name, 'dids': dids, 'rse_id': rse_id}], account=account, ignore_duplicate=ignore_duplicate, session=session)
 
 
 @transactional_session
@@ -340,6 +341,7 @@ def attach_dids_to_dids(
                                                collections_temp_table=children_temp_table,
                                                collections=children,
                                                account=account,
+                                               ignore_duplicate=ignore_duplicate,
                                                session=session)
                 update_parent = True
 
@@ -669,6 +671,7 @@ def __add_collections_to_container(
     collections_temp_table: Any,
     collections: "Mapping[tuple[InternalScope, str], Mapping[str, Any]]",
     account: "InternalAccount",
+    ignore_duplicate: bool = False,
     *,
     session: "Session"
 ) -> None:
@@ -678,6 +681,7 @@ def __add_collections_to_container(
     :param parent_did: the DataIdentifier object of the parent did
     :param collections: .
     :param account: The account owner.
+    :param ignore_duplicate: If True, ignore duplicate entries.
     :param session: The database session in use.
     """
 
@@ -695,9 +699,27 @@ def __add_collections_to_container(
         and_(models.DataIdentifier.scope == collections_temp_table.scope,
              models.DataIdentifier.name == collections_temp_table.name),
     )
+    if ignore_duplicate:
+        stmt = stmt.add_columns(
+            models.DataIdentifierAssociation.scope.label("ignore_duplicate_parent_scope"),
+            models.DataIdentifierAssociation.name.label("ignore_duplicate_parent_name"),
+            collections_temp_table.scope.label("ignore_duplicate_child_scope"),
+            collections_temp_table.name.label("ignore_duplicate_child_name"),
+        ).outerjoin_from(
+            collections_temp_table,
+            models.DataIdentifierAssociation,
+            and_(
+                models.DataIdentifierAssociation.scope == parent_did.scope,
+                models.DataIdentifierAssociation.name == parent_did.name,
+                models.DataIdentifierAssociation.child_scope == collections_temp_table.scope,
+                models.DataIdentifierAssociation.child_name == collections_temp_table.name,
+            ),
+        )
 
     container_parents = None
     child_type = None
+    ignore_duplicate_existing_attachments = set()
+
     for row in session.execute(stmt):
 
         if row.did_scope is None:
@@ -719,8 +741,14 @@ def __add_collections_to_container(
             if (row.scope, row.name) in container_parents:
                 raise exception.UnsupportedOperation('Circular attachment detected. %s:%s is already a parent of %s:%s' % (row.scope, row.name, parent_did.scope, parent_did.name))
 
+        if ignore_duplicate and row.ignore_duplicate_parent_scope is not None:
+            ignore_duplicate_existing_attachments.add((row.ignore_duplicate_parent_scope, row.ignore_duplicate_parent_name, row.ignore_duplicate_child_scope, row.ignore_duplicate_child_name))
+
     messages = []
     for c in collections.values():
+        if ignore_duplicate and (parent_did.scope, parent_did.name, c['scope'], c['name']) in ignore_duplicate_existing_attachments:
+            continue
+
         did_asso = models.DataIdentifierAssociation(
             scope=parent_did.scope,
             name=parent_did.name,
@@ -1320,7 +1348,7 @@ def list_new_dids(
             select_stmt = select_stmt.where(
                 models.DataIdentifier.did_type == DIDType[did_type]
             )
-        elif isinstance(did_type, Enum):
+        elif isinstance(did_type, DIDType):
             select_stmt = select_stmt.where(
                 models.DataIdentifier.did_type == did_type
             )
@@ -2268,7 +2296,7 @@ def set_status(
 @read_session
 def list_dids(
     scope: "InternalScope",
-    filters: "Mapping[Any, Any]",
+    filters: "Iterable[dict[Any, Any]]",
     did_type: Literal['all', 'collection', 'dataset', 'container', 'file'] = 'collection',
     ignore_case: bool = False,
     limit: Optional[int] = None,

rucio/core/did_meta_plugins/did_column_meta.py

@@ -234,7 +234,7 @@ class DidColumnMeta(DidMetaPlugin):
         )
         stmt = stmt.with_hint(
             models.DataIdentifier,
-            'NO_EXPAND',
+            'USE_CONCAT INDEX_RS_ASC(DIDS)',
             'oracle'
         )
 

rucio/core/replica.py

@@ -243,7 +243,7 @@ def __exist_replicas(
 
 @read_session
 def list_bad_replicas_status(
-    state: BadFilesStatus = BadFilesStatus.BAD,
+    state: Optional[BadFilesStatus] = BadFilesStatus.BAD,
     rse_id: Optional[str] = None,
     younger_than: Optional[datetime] = None,
     older_than: Optional[datetime] = None,

rucio/db/sqla/util.py

@@ -337,7 +337,7 @@ def list_oracle_global_temp_tables(session: "Session") -> list[str]:
         global_temp_tables = [
             str(t[0]).upper()
             for t in session.execute(
-                text('SELECT UPPER(table_name) '
+                text("SELECT /*+ OPT_PARAM('OPTIMIZER_FEATURES_ENABLE', '11.2.0.4') */ UPPER(table_name) "
                      'FROM all_tables '
                      'WHERE OWNER = :owner AND IOT_NAME IS NULL AND DURATION IS NOT NULL'),
                 dict(owner=models.BASE.metadata.schema.upper())

rucio/gateway/authentication.py

@@ -12,27 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import Any, Optional
 
 from rucio.common import exception
 from rucio.common.types import InternalAccount, TokenDict
 from rucio.common.utils import gateway_update_return_dict
 from rucio.core import authentication, identity, oidc
-from rucio.db.sqla.constants import IdentityType
-from rucio.db.sqla.session import transactional_session
+from rucio.db.sqla.constants import DatabaseOperationType, IdentityType
+from rucio.db.sqla.session import db_session
 from rucio.gateway import permission
 
-if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
 
-
-@transactional_session
 def refresh_cli_auth_token(
     token_string: str,
     account: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[tuple[str, int]]:
     """
     Checks if there is active refresh token and if so returns
@@ -40,20 +34,18 @@ def refresh_cli_auth_token(
     refresh and returns new access token.
     :param token_string: token string
     :param account: Rucio account for which token refresh should be considered
-    :param session: The database session in use.
 
     :return: tuple of (access token, expiration epoch), None otherswise
     """
     internal_account = InternalAccount(account, vo=vo)
-    return oidc.refresh_cli_auth_token(token_string, internal_account, session=session)
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return oidc.refresh_cli_auth_token(token_string, internal_account, session=session)
 
 
-@transactional_session
 def redirect_auth_oidc(
     authn_code: str,
     fetchtoken: bool = False,
-    *,
-    session: "Session"
 ) -> Optional[str]:
     """
     Finds the Authentication URL in the Rucio DB oauth_requests table
@@ -63,21 +55,18 @@ def redirect_auth_oidc(
                       authorization securely to IdP via Rucio Auth server through a browser.
     :param fetchtoken: If True, valid token temporarily saved in the oauth_requests table
                        will be returned. If False, redirection URL is returned.
-    :param session: The database session in use.
 
     :returns: result of the query (authorization URL or a
               token if a user asks with the correct code) or None.
               Exception thrown in case of an unexpected crash.
     """
-    return authentication.redirect_auth_oidc(authn_code, fetchtoken, session=session)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return authentication.redirect_auth_oidc(authn_code, fetchtoken, session=session)
 
 
-@transactional_session
 def get_auth_oidc(
     account: str,
     vo: str = 'def',
-    *,
-    session: "Session",
     **kwargs
 ) -> str:
     """
@@ -107,7 +96,6 @@ def get_auth_oidc(
     :param refresh_lifetime: specifies how long the OAuth daemon should
                              be refreshing this token. Default is 96 hours.
     :param ip: IP address of the client as a string.
-    :param session: The database session in use.
 
     :returns: User & Rucio OIDC Client specific Authorization or Redirection URL as a string
               OR a redirection url to be used in user's browser for authentication.
@@ -115,16 +103,14 @@ def get_auth_oidc(
     # no permission layer for the moment !
 
     internal_account = InternalAccount(account, vo=vo)
-    return oidc.get_auth_oidc(internal_account, session=session, **kwargs)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return oidc.get_auth_oidc(internal_account, session=session, **kwargs)
 
 
-@transactional_session
 def get_token_oidc(
     auth_query_string: str,
     ip: Optional[str] = None,
-    *,
-    session: "Session"
-) -> Optional[dict[str, Optional[Union[str, bool]]]]:
+) -> Optional[dict[str, Any]]:
     """
     After Rucio User got redirected to Rucio /auth/oidc_token (or /auth/oidc_code)
     REST endpoints with authz code and session state encoded within the URL.
@@ -132,17 +118,16 @@ def get_token_oidc(
 
     :param auth_query_string: IdP redirection URL query string (AuthZ code & user session state).
     :param ip: IP address of the client as a string.
-    :param session: The database session in use.
 
     :returns: One of the following tuples: ("fetchcode", <code>); ("token", <token>);
               ("polling", True); The result depends on the authentication strategy being used
               (no auto, auto, polling).
     """
     # no permission layer for the moment !
-    return oidc.get_token_oidc(auth_query_string, ip, session=session)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return oidc.get_token_oidc(auth_query_string, ip, session=session)
 
 
-@transactional_session
 def get_auth_token_user_pass(
     account: str,
     username: str,
@@ -150,8 +135,6 @@ def get_auth_token_user_pass(
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Authenticate a Rucio account temporarily via username and password.
@@ -164,30 +147,27 @@ def get_auth_token_user_pass(
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
 
     kwargs = {'account': account, 'username': username, 'password': password}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_user_pass', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (username, account, auth_result.message))
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_user_pass', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (username, account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_auth_token_user_pass(internal_account, username, password, appid, ip, session=session)
+        return authentication.get_auth_token_user_pass(internal_account, username, password, appid, ip, session=session)
 
 
-@transactional_session
 def get_auth_token_gss(
     account: str,
     gsscred: str,
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Authenticate a Rucio account temporarily via a GSS token.
@@ -199,30 +179,27 @@ def get_auth_token_gss(
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
 
     kwargs = {'account': account, 'gsscred': gsscred}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_gss', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (gsscred, account, auth_result.message))
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_gss', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (gsscred, account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_auth_token_gss(internal_account, gsscred, appid, ip, session=session)
+        return authentication.get_auth_token_gss(internal_account, gsscred, appid, ip, session=session)
 
 
-@transactional_session
 def get_auth_token_x509(
     account: Optional[str],
     dn: str,
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Authenticate a Rucio account temporarily via an x509 certificate.
@@ -234,7 +211,6 @@ def get_auth_token_x509(
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
@@ -243,24 +219,23 @@ def get_auth_token_x509(
         account = identity.get_default_account(dn, IdentityType.X509).external
 
     kwargs = {'account': account, 'dn': dn}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_x509', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (dn, account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_x509', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (dn, account, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_auth_token_x509(internal_account, dn, appid, ip, session=session)
+        return authentication.get_auth_token_x509(internal_account, dn, appid, ip, session=session)
 
 
-@transactional_session
 def get_auth_token_ssh(
     account: str,
     signature: str,
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Authenticate a Rucio account temporarily via SSH key exchange.
@@ -272,29 +247,27 @@ def get_auth_token_ssh(
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
 
     kwargs = {'account': account, 'signature': signature}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_ssh', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User with provided signature can not log to account %s. %s' % (account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_ssh', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User with provided signature can not log to account %s. %s' % (account, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_auth_token_ssh(internal_account, signature, appid, ip, session=session)
+        return authentication.get_auth_token_ssh(internal_account, signature, appid, ip, session=session)
 
 
-@transactional_session
 def get_ssh_challenge_token(
     account: str,
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Get a challenge token for subsequent SSH public key authentication.
@@ -305,30 +278,28 @@ def get_ssh_challenge_token(
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
 
     kwargs = {'account': account}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_ssh', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User can not get challenge token for account %s. %s' % (account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_ssh', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User can not get challenge token for account %s. %s' % (account, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_ssh_challenge_token(internal_account, appid, ip, session=session)
+        return authentication.get_ssh_challenge_token(internal_account, appid, ip, session=session)
 
 
-@transactional_session
 def get_auth_token_saml(
     account: str,
     saml_nameid: str,
     appid: str,
     ip: Optional[str] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> Optional[TokenDict]:
     """
     Authenticate a Rucio account temporarily via SSO.
@@ -339,26 +310,24 @@ def get_auth_token_saml(
     :param saml_nameid: NameId returned in SAML response as a string.
     :param appid: The application identifier as a string.
     :param ip: IP address of the client as a string.
-    :param session: The database session in use.
 
     :returns: A dict with token and expires_at entries.
     """
 
     kwargs = {'account': account, 'saml_nameid': saml_nameid}
-    auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_saml', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (saml_nameid, account, auth_result.message))
 
-    internal_account = InternalAccount(account, vo=vo)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=account, vo=vo, action='get_auth_token_saml', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('User with identity %s can not log to account %s. %s' % (saml_nameid, account, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
 
-    return authentication.get_auth_token_saml(internal_account, saml_nameid, appid, ip, session=session)
+        return authentication.get_auth_token_saml(internal_account, saml_nameid, appid, ip, session=session)
 
 
-@transactional_session
 def validate_auth_token(
     token: str,
-    *,
-    session: "Session"
 ) -> dict[str, Any]:
     """
     Validate an authentication token.
@@ -374,8 +343,9 @@ def validate_auth_token(
                            vo: <vo> }
     """
 
-    auth = authentication.validate_auth_token(token, session=session)
-    vo = auth['account'].vo
-    auth = gateway_update_return_dict(auth, session=session)
-    auth['vo'] = vo
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth = authentication.validate_auth_token(token, session=session)
+        vo = auth['account'].vo
+        auth = gateway_update_return_dict(auth, session=session)
+        auth['vo'] = vo
     return auth