rucio 37.2.0__py3-none-any.whl → 37.4.0__py3-none-any.whl

rucio/gateway/dirac.py

@@ -15,28 +15,25 @@
 from typing import TYPE_CHECKING, Any, Optional
 
 from rucio.common.exception import AccessDenied
+from rucio.common.types import InternalScope
 from rucio.common.utils import extract_scope
 from rucio.core import dirac
 from rucio.core.rse import get_rse_id
-from rucio.db.sqla.session import transactional_session
+from rucio.core.scope import list_scopes
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
 from rucio.gateway.permission import has_permission
-from rucio.gateway.scope import list_scopes
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from sqlalchemy.orm import Session
 
-
-@transactional_session
 def add_files(
     lfns: "Iterable[dict[str, Any]]",
     issuer: str,
     ignore_availability: bool,
     parents_metadata: Optional[dict[str, Any]] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> None:
     """
     Bulk add files :
@@ -49,35 +46,37 @@ def add_files(
     :param ignore_availability: A boolean to ignore blocked sites.
     :param parents_metadata: Metadata for selected hierarchy DIDs. (dictionary {'lpn': {key : value}}). Default=None
     :param vo: The VO to act on.
-    :param session: The database session in use.
 
     """
-    scopes = list_scopes(vo=vo, session=session)
-    dids = []
-    rses = {}
-    for lfn in lfns:
-        scope, name = extract_scope(lfn['lfn'], scopes)
-        dids.append({'scope': scope, 'name': name})
-        rse = lfn['rse']
-        if rse not in rses:
-            rse_id = get_rse_id(rse=rse, vo=vo, session=session)
-            rses[rse] = rse_id
-        lfn['rse_id'] = rses[rse]
 
-    # Check if the issuer can add dids and use skip_availabitlity
-    for rse in rses:
-        rse_id = rses[rse]
-        kwargs = {'rse': rse, 'rse_id': rse_id}
-        auth_result = has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo, session=session)
-        if not auth_result.allowed:
-            raise AccessDenied('Account %s can not add file replicas on %s for VO %s. %s' % (issuer, rse, vo, auth_result.message))
-        if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs, vo=vo, session=session):
-            ignore_availability = False
+    with db_session(DatabaseOperationType.WRITE) as session:
+        filter_ = {'scope': InternalScope(scope='*', vo=vo)}
+        scopes = [scope.external for scope in list_scopes(filter_=filter_, session=session)]
+        dids = []
+        rses = {}
+        for lfn in lfns:
+            scope, name = extract_scope(lfn['lfn'], scopes)
+            dids.append({'scope': scope, 'name': name})
+            rse = lfn['rse']
+            if rse not in rses:
+                rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+                rses[rse] = rse_id
+            lfn['rse_id'] = rses[rse]
 
-    # Check if the issuer can add the files
-    kwargs = {'issuer': issuer, 'dids': dids}
-    auth_result = has_permission(issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo, session=session)
-    if not auth_result.allowed:
-        raise AccessDenied('Account %s can not bulk add data identifier for VO %s. %s' % (issuer, vo, auth_result.message))
+        # Check if the issuer can add dids and use skip_availabitlity
+        for rse in rses:
+            rse_id = rses[rse]
+            kwargs = {'rse': rse, 'rse_id': rse_id}
+            auth_result = has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo, session=session)
+            if not auth_result.allowed:
+                raise AccessDenied('Account %s can not add file replicas on %s for VO %s. %s' % (issuer, rse, vo, auth_result.message))
+            if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs, vo=vo, session=session):
+                ignore_availability = False
+
+        # Check if the issuer can add the files
+        kwargs = {'issuer': issuer, 'dids': dids}
+        auth_result = has_permission(issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo, session=session)
+        if not auth_result.allowed:
+            raise AccessDenied('Account %s can not bulk add data identifier for VO %s. %s' % (issuer, vo, auth_result.message))
 
-    dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, parents_metadata=parents_metadata, vo=vo, session=session)
+        dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, parents_metadata=parents_metadata, vo=vo, session=session)

rucio/gateway/exporter.py

@@ -12,49 +12,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from rucio.common import exception
 from rucio.core import exporter
 from rucio.core.rse import get_rse_name
-from rucio.db.sqla.session import read_session
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
 from rucio.gateway import permission
 
-if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
 
-
-@read_session
-def export_data(issuer: str, distance: bool = True, vo: str = 'def', *, session: "Session") -> dict[str, Any]:
+def export_data(issuer: str, distance: bool = True, vo: str = 'def') -> dict[str, Any]:
     """
     Export data from Rucio.
 
     :param issuer: the issuer.
     :param distance: To enable the reporting of distance.
     :param vo: the VO of the issuer.
-    :param session: The database session in use.
     """
     kwargs = {'issuer': issuer}
-    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='export', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('Account %s can not export data. %s' % (issuer, auth_result.message))
-
-    data = exporter.export_data(distance=distance, vo=vo, session=session)
-    rses = {}
-    distances = {}
-
-    for rse_id in data['rses']:
-        rse = data['rses'][rse_id]
-        rses[get_rse_name(rse_id=rse_id, session=session)] = rse
-    data['rses'] = rses
-
-    if distance:
-        for src_id in data['distances']:
-            dests = data['distances'][src_id]
-            src = get_rse_name(rse_id=src_id, session=session)
-            distances[src] = {}
-            for dest_id in dests:
-                dest = get_rse_name(rse_id=dest_id, session=session)
-                distances[src][dest] = dests[dest_id]
-        data['distances'] = distances
+    with db_session(DatabaseOperationType.READ) as session:
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='export', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not export data. %s' % (issuer, auth_result.message))
+
+        data = exporter.export_data(distance=distance, vo=vo, session=session)
+        rses = {}
+        distances = {}
+
+        for rse_id in data['rses']:
+            rse = data['rses'][rse_id]
+            rses[get_rse_name(rse_id=rse_id, session=session)] = rse
+        data['rses'] = rses
+
+        if distance:
+            for src_id in data['distances']:
+                dests = data['distances'][src_id]
+                src = get_rse_name(rse_id=src_id, session=session)
+                distances[src] = {}
+                for dest_id in dests:
+                    dest = get_rse_name(rse_id=dest_id, session=session)
+                    distances[src][dest] = dests[dest_id]
+            data['distances'] = distances
     return data

rucio/gateway/importer.py

@@ -12,35 +12,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from rucio.common import exception
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalAccount
 from rucio.core import importer
-from rucio.db.sqla.session import transactional_session
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
 from rucio.gateway import permission
 
-if TYPE_CHECKING:
-    from sqlalchemy.orm import Session
 
-
-@transactional_session
-def import_data(data: dict[str, Any], issuer: str, vo: str = 'def', *, session: "Session") -> None:
+def import_data(data: dict[str, Any], issuer: str, vo: str = 'def') -> None:
     """
     Import data to add/update/delete records in Rucio.
 
     :param data: data to be imported.
     :param issuer: the issuer.
     :param vo: the VO of the issuer.
-    :param session: The database session in use.
     """
     kwargs = {'issuer': issuer}
     validate_schema(name='import', obj=data, vo=vo)
-    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('Account %s can not import data. %s' % (issuer, auth_result.message))
 
-    for account in data.get('accounts', []):
-        account['account'] = InternalAccount(account['account'], vo=vo)
-    return importer.import_data(data, vo=vo, session=session)
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not import data. %s' % (issuer, auth_result.message))
+
+        for account in data.get('accounts', []):
+            account['account'] = InternalAccount(account['account'], vo=vo)
+        return importer.import_data(data, vo=vo, session=session)

rucio/gateway/lifetime_exception.py

@@ -18,24 +18,20 @@ from rucio.common import exception
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import gateway_update_return_dict
 from rucio.core import lifetime_exception
-from rucio.db.sqla.session import stream_session, transactional_session
+from rucio.db.sqla.constants import DatabaseOperationType
+from rucio.db.sqla.session import db_session
 from rucio.gateway import permission
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
 
-    from sqlalchemy.orm import Session
-
     from rucio.db.sqla.constants import LifetimeExceptionsState
 
 
-@stream_session
 def list_exceptions(
     exception_id: Optional[str] = None,
     states: Optional["Iterable[LifetimeExceptionsState]"] = None,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     List exceptions to Lifetime Model.
@@ -43,16 +39,15 @@ def list_exceptions(
     :param id:         The id of the exception
     :param states:     The states to filter
     :param vo:         The VO to act on
-    :param session:    The database session in use.
     """
 
-    exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states, session=session)
-    for e in exceptions:
-        if vo == e['scope'].vo:
-            yield gateway_update_return_dict(e, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states, session=session)
+        for e in exceptions:
+            if vo == e['scope'].vo:
+                yield gateway_update_return_dict(e, session=session)
 
 
-@transactional_session
 def add_exception(
     dids: "Iterable[dict[str, Any]]",
     account: str,
@@ -60,8 +55,6 @@ def add_exception(
     comments: str,
     expires_at: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> dict[str, Any]:
     """
     Add exceptions to Lifetime Model.
@@ -72,7 +65,6 @@ def add_exception(
     :param comments:    The comments associated to the exception.
     :param expires_at:  The expiration date of the exception.
     :param vo:          The VO to act on.
-    :param session:     The database session in use.
 
     returns:            The id of the exception.
     """
@@ -80,7 +72,9 @@ def add_exception(
     internal_account = InternalAccount(account, vo=vo)
     for did in dids:
         did['scope'] = InternalScope(did['scope'], vo=vo)
-    exceptions = lifetime_exception.add_exception(dids=dids, account=internal_account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        exceptions = lifetime_exception.add_exception(dids=dids, account=internal_account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
 
     for key in exceptions:
         if key == 'exceptions':
@@ -96,14 +90,11 @@ def add_exception(
     return exceptions
 
 
-@transactional_session
 def update_exception(
     exception_id: str,
     state: 'LifetimeExceptionsState',
     issuer: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> None:
     """
     Update exceptions state to Lifetime Model.
@@ -112,10 +103,11 @@ def update_exception(
     :param state:      The states to filter.
     :param issuer:     The issuer account.
     :param vo:         The VO to act on.
-    :param session:    The database session in use.
     """
     kwargs = {'exception_id': exception_id, 'vo': vo}
-    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_lifetime_exceptions', kwargs=kwargs, session=session)
-    if not auth_result.allowed:
-        raise exception.AccessDenied('Account %s can not update lifetime exceptions. %s' % (issuer, auth_result.message))
-    return lifetime_exception.update_exception(exception_id=exception_id, state=state, session=session)
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_lifetime_exceptions', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not update lifetime exceptions. %s' % (issuer, auth_result.message))
+        return lifetime_exception.update_exception(exception_id=exception_id, state=state, session=session)

rucio/gateway/lock.py

@@ -19,26 +19,21 @@ from rucio.common.types import InternalScope
 from rucio.common.utils import gateway_update_return_dict
 from rucio.core import lock
 from rucio.core.rse import get_rse_id
-from rucio.db.sqla.constants import DIDType
-from rucio.db.sqla.session import stream_session
+from rucio.db.sqla.constants import DatabaseOperationType, DIDType
+from rucio.db.sqla.session import db_session
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
 
-    from sqlalchemy.orm import Session
-
 
 LOGGER = logging.getLogger('lock')
 LOGGER.setLevel(logging.DEBUG)
 
 
-@stream_session
 def get_dataset_locks(
     scope: str,
     name: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     Get the dataset locks of a dataset.
@@ -46,24 +41,21 @@ def get_dataset_locks(
     :param scope:          Scope of the dataset.
     :param name:           Name of the dataset.
     :param vo:             The VO to act on.
-    :param session:        The database session in use.
     :return:               List of dicts {'rse_id': ..., 'state': ...}
     """
 
     internal_scope = InternalScope(scope, vo=vo)
 
-    locks = lock.get_dataset_locks(scope=internal_scope, name=name, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        locks = lock.get_dataset_locks(scope=internal_scope, name=name, session=session)
 
-    for lock_object in locks:
-        yield gateway_update_return_dict(lock_object, session=session)
+        for lock_object in locks:
+            yield gateway_update_return_dict(lock_object, session=session)
 
 
-@stream_session
 def get_dataset_locks_bulk(
     dids: 'Iterable[dict[str, Any]]',
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     Get the dataset locks for multiple datasets or containers.
@@ -72,7 +64,6 @@ def get_dataset_locks_bulk(
                             "type" is optional. If present, will be either DIDType.DATASET or DIDType.CONTAINER,
                             or string "dataset" or "container"
     :param vo:              The VO to act on.
-    :param session:         The database session in use.
     :return:                Generator of dicts describing found locks {'rse_id': ..., 'state': ...}. Duplicates are removed
     """
 
@@ -96,58 +87,54 @@ def get_dataset_locks_bulk(
         dids_converted.append(did)
 
     seen = set()
-    for lock_info in lock.get_dataset_locks_bulk(dids_converted, session=session):
-        # filter duplicates - same scope, name, rse_id, rule_id
-        scope_str = str(lock_info["scope"])
-        key = (scope_str, lock_info["name"], lock_info["rse_id"], lock_info["rule_id"])
-        if key not in seen:
-            seen.add(key)
-            yield lock_info
+
+    with db_session(DatabaseOperationType.READ) as session:
+        for lock_info in lock.get_dataset_locks_bulk(dids_converted, session=session):
+            # filter duplicates - same scope, name, rse_id, rule_id
+            scope_str = str(lock_info["scope"])
+            key = (scope_str, lock_info["name"], lock_info["rse_id"], lock_info["rule_id"])
+            if key not in seen:
+                seen.add(key)
+                yield lock_info
 
 
-@stream_session
 def get_dataset_locks_by_rse(
     rse: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     Get the dataset locks of an RSE.
 
     :param rse:            RSE name.
     :param vo:             The VO to act on.
-    :param session:        The database session in use.
     :return:               List of dicts {'rse_id': ..., 'state': ...}
     """
 
-    rse_id = get_rse_id(rse=rse, vo=vo, session=session)
-    locks = lock.get_dataset_locks_by_rse_id(rse_id=rse_id, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+        locks = lock.get_dataset_locks_by_rse_id(rse_id=rse_id, session=session)
 
-    for lock_object in locks:
-        yield gateway_update_return_dict(lock_object, session=session)
+        for lock_object in locks:
+            yield gateway_update_return_dict(lock_object, session=session)
 
 
-@stream_session
 def get_replica_locks_for_rule_id(
     rule_id: str,
     vo: str = 'def',
-    *,
-    session: "Session"
 ) -> 'Iterator[dict[str, Any]]':
     """
     Get the replica locks for a rule_id.
 
     :param rule_id:     Rule ID.
     :param vo:          The VO to act on.
-    :param session:     The database session in use.
     :return:            List of dicts.
     """
 
-    locks = lock.get_replica_locks_for_rule_id(rule_id=rule_id, session=session)
+    with db_session(DatabaseOperationType.READ) as session:
+        locks = lock.get_replica_locks_for_rule_id(rule_id=rule_id, session=session)
 
-    for lock_object in locks:
-        if lock_object['scope'].vo != vo:  # rule is on a different VO, so don't return any locks
-            LOGGER.debug('rule id %s is not present on VO %s' % (rule_id, vo))
-            break
-        yield gateway_update_return_dict(lock_object, session=session)
+        for lock_object in locks:
+            if lock_object['scope'].vo != vo:  # rule is on a different VO, so don't return any locks
+                LOGGER.debug('rule id %s is not present on VO %s' % (rule_id, vo))
+                break
+            yield gateway_update_return_dict(lock_object, session=session)