rucio 37.0.0rc1__py3-none-any.whl

rucio/gateway/lock.py

@@ -0,0 +1,153 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from rucio.common.types import InternalScope
+from rucio.common.utils import gateway_update_return_dict
+from rucio.core import lock
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.constants import DIDType
+from rucio.db.sqla.session import stream_session
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from sqlalchemy.orm import Session
+
+
+LOGGER = logging.getLogger('lock')
+LOGGER.setLevel(logging.DEBUG)
+
+
+@stream_session
+def get_dataset_locks(
+    scope: str,
+    name: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    Get the dataset locks of a dataset.
+
+    :param scope:          Scope of the dataset.
+    :param name:           Name of the dataset.
+    :param vo:             The VO to act on.
+    :param session:        The database session in use.
+    :return:               List of dicts {'rse_id': ..., 'state': ...}
+    """
+
+    internal_scope = InternalScope(scope, vo=vo)
+
+    locks = lock.get_dataset_locks(scope=internal_scope, name=name, session=session)
+
+    for lock_object in locks:
+        yield gateway_update_return_dict(lock_object, session=session)
+
+
+@stream_session
+def get_dataset_locks_bulk(
+    dids: 'Iterable[dict[str, Any]]',
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    Get the dataset locks for multiple datasets or containers.
+
+    :param dids:            List of dataset or container DIDs as dictionaries {"scope":..., "name":..., "type":...}
+                            "type" is optional. If present, will be either DIDType.DATASET or DIDType.CONTAINER,
+                            or string "dataset" or "container"
+    :param vo:              The VO to act on.
+    :param session:         The database session in use.
+    :return:                Generator of dicts describing found locks {'rse_id': ..., 'state': ...}. Duplicates are removed
+    """
+
+    if vo is None:
+        vo = "def"
+
+    dids_converted = []
+    for did_in in dids:
+        did = did_in.copy()
+        if isinstance(did.get("type"), str):
+            # convert DID type
+            try:
+                did["type"] = {
+                    "dataset": DIDType.DATASET,
+                    "container": DIDType.CONTAINER
+                }[did["type"]]
+            except KeyError:
+                raise ValueError("Unknown DID type %(type)s" % did)
+        if isinstance(did["scope"], str):
+            did["scope"] = InternalScope(did["scope"], vo=vo)
+        dids_converted.append(did)
+
+    seen = set()
+    for lock_info in lock.get_dataset_locks_bulk(dids_converted, session=session):
+        # filter duplicates - same scope, name, rse_id, rule_id
+        scope_str = str(lock_info["scope"])
+        key = (scope_str, lock_info["name"], lock_info["rse_id"], lock_info["rule_id"])
+        if key not in seen:
+            seen.add(key)
+            yield lock_info
+
+
+@stream_session
+def get_dataset_locks_by_rse(
+    rse: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    Get the dataset locks of an RSE.
+
+    :param rse:            RSE name.
+    :param vo:             The VO to act on.
+    :param session:        The database session in use.
+    :return:               List of dicts {'rse_id': ..., 'state': ...}
+    """
+
+    rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+    locks = lock.get_dataset_locks_by_rse_id(rse_id=rse_id, session=session)
+
+    for lock_object in locks:
+        yield gateway_update_return_dict(lock_object, session=session)
+
+
+@stream_session
+def get_replica_locks_for_rule_id(
+    rule_id: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    Get the replica locks for a rule_id.
+
+    :param rule_id:     Rule ID.
+    :param vo:          The VO to act on.
+    :param session:     The database session in use.
+    :return:            List of dicts.
+    """
+
+    locks = lock.get_replica_locks_for_rule_id(rule_id=rule_id, session=session)
+
+    for lock_object in locks:
+        if lock_object['scope'].vo != vo:  # rule is on a different VO, so don't return any locks
+            LOGGER.debug('rule id %s is not present on VO %s' % (rule_id, vo))
+            break
+        yield gateway_update_return_dict(lock_object, session=session)

rucio/gateway/meta_conventions.py

@@ -0,0 +1,98 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Optional, Union
+
+from rucio.common.exception import AccessDenied
+from rucio.core import meta_conventions
+from rucio.db.sqla.session import read_session, transactional_session
+from rucio.gateway.permission import has_permission
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+    from rucio.common.types import InternalAccount
+    from rucio.db.sqla.constants import KeyType
+
+
+@read_session
+def list_keys(*, session: "Session") -> list[str]:
+    """
+    Lists all keys for DID Metadata Conventions.
+
+    :param session: The database session in use.
+
+    :returns: A list containing all keys.
+    """
+    return meta_conventions.list_keys(session=session)
+
+
+@read_session
+def list_values(key: str, *, session: "Session") -> list[str]:
+    """
+    Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
+
+    :param key: the name for the key.
+    :param session: The database session in use.
+
+
+    :returns: A list containing all values.
+    """
+    return meta_conventions.list_values(key=key, session=session)
+
+
+@transactional_session
+def add_key(
+    key: str,
+    key_type: Union["KeyType", str],
+    issuer: "InternalAccount",
+    value_type: Optional[str] = None,
+    value_regexp: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
+
+    :param key: the name for the new key.
+    :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
+    :param issuer: The issuer account.
+    :param value_type: the type of the value, if defined.
+    :param value_regexp: the regular expression that values should match, if defined.
+    :param vo: The vo to act on
+    :param session: The database session in use.
+    """
+    kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp}
+    auth_result = has_permission(issuer=issuer, vo=vo, action='add_key', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise AccessDenied('Account %s can not add key. %s' % (issuer, auth_result.message))
+    return meta_conventions.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
+
+
+@transactional_session
+def add_value(key: str, value: str, issuer: "InternalAccount", vo: str = 'def', *, session: "Session") -> None:
+    """
+    Add an allowed value for DID metadata (update a key in DID Metadata Conventions table).
+
+    :param key: the name for the key.
+    :param value: the value.
+    :param vo: the vo to act on.
+    :param session: The database session in use.
+    """
+    kwargs = {'key': key, 'value': value}
+    auth_result = has_permission(issuer=issuer, vo=vo, action='add_value', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise AccessDenied('Account %s can not add value %s to key %s. %s' % (issuer, value, key, auth_result.message))
+    return meta_conventions.add_value(key=key, value=value, session=session)

rucio/gateway/permission.py

@@ -0,0 +1,74 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any
+
+from rucio.common.exception import RSENotFound
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.core import permission
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.session import read_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+    from rucio.core.permission import PermissionResult
+
+
+@read_session
+def has_permission(issuer: str, action: str, kwargs: dict[str, Any], vo: str = 'def', *, session: "Session") -> 'PermissionResult':
+    """
+    Checks if an account has the specified permission to
+    execute an action with parameters.
+
+    :param issuer:  The Account issuer.
+    :param vo:      The VO to check against.
+    :param action:  The action (API call) called by the account.
+    :param session: The db session to use
+    :param kwargs:  List of arguments for the action.
+    :returns: True if account is allowed to call the API call, otherwise False
+    """
+
+    kwargs = deepcopy(kwargs)
+    if 'rse' in kwargs and 'rse_id' not in kwargs:
+        try:
+            rse_id = get_rse_id(rse=kwargs.get('rse'), vo=vo, session=session)
+        except RSENotFound:
+            rse_id = None
+        kwargs.update({'rse_id': rse_id})
+
+    if 'scope' in kwargs:
+        kwargs['scope'] = InternalScope(kwargs['scope'], vo=vo)
+    if 'attachments' in kwargs:
+        for a in kwargs['attachments']:
+            a['scope'] = InternalScope(a['scope'], vo=vo)
+
+    if 'account' in kwargs:
+        kwargs['account'] = InternalAccount(kwargs['account'], vo=vo)
+    if 'accounts' in kwargs:
+        kwargs['accounts'] = [InternalAccount(a, vo=vo) for a in kwargs['accounts']]
+    if 'rules' in kwargs:
+        for r in kwargs['rules']:
+            r['account'] = InternalAccount(r['account'], vo=vo)
+    if 'dids' in kwargs:
+        for d in kwargs['dids']:
+            d['scope'] = InternalScope(d['scope'], vo=vo)
+            if 'rules' in d:
+                for r in d['rules']:
+                    r['account'] = InternalAccount(r['account'], vo=vo)
+
+    issuer_account = InternalAccount(issuer, vo=vo)
+
+    return permission.has_permission(issuer=issuer_account, action=action, kwargs=kwargs, session=session)

rucio/gateway/quarantined_replica.py

@@ -0,0 +1,79 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common import exception
+from rucio.common.types import InternalScope
+from rucio.core.quarantined_replica import add_quarantined_replicas
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.session import transactional_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def quarantine_file_replicas(
+    replicas: "Iterable[dict[str, Any]]",
+    issuer: str,
+    rse: Optional[str] = None,
+    rse_id: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Quarantine replicas.
+
+    :param replicas: List of replica infos: {'scope': <scope> (optional), 'name': <name> (optional), 'path':<path> (required)}
+    :param issuer: The issuer account.
+    :param vo: The VO to act on.
+    :param rse: RSE name
+    :param rse_id: RSE id - either RSE name or RSE id must be specified
+    :param session: The database session in use.
+    """
+
+    if not replicas:
+        return
+
+    if (rse is None) == (rse_id is None):
+        raise exception.InputValidationError("Either RSE name or RSE id must be specified, but not both")
+
+    if rse_id is None:
+        rse_id = get_rse_id(rse, vo=vo, session=session)
+
+    auth_result = permission.has_permission(issuer, 'quarantine_file_replicas', {}, vo=vo, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account %s can not quarantine replicas. %s' % (issuer, auth_result.message))
+
+    replica_infos = []
+    for r in replicas:
+        if "path" not in r:
+            raise exception.InputValidationError("Replica info must include path")
+        scope = r.get("scope")
+        if scope and isinstance(scope, str):
+            scope = InternalScope(scope, vo=vo)
+        replica_infos.append(
+            {
+                "scope": scope or None,
+                "name": r.get("name"),
+                "path": r["path"]
+            }
+        )
+
+    add_quarantined_replicas(rse_id, replica_infos, session=session)