rucio 37.0.0rc1__py3-none-any.whl

rucio/gateway/dirac.py

@@ -0,0 +1,83 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common.exception import AccessDenied
+from rucio.common.utils import extract_scope
+from rucio.core import dirac
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.session import transactional_session
+from rucio.gateway.permission import has_permission
+from rucio.gateway.scope import list_scopes
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_files(
+    lfns: "Iterable[dict[str, Any]]",
+    issuer: str,
+    ignore_availability: bool,
+    parents_metadata: Optional[dict[str, Any]] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Bulk add files :
+    - Create the file and replica.
+    - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
+    - Create all the ascendants of the dataset if they do not exist
+
+    :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
+    :param issuer: The issuer account.
+    :param ignore_availability: A boolean to ignore blocked sites.
+    :param parents_metadata: Metadata for selected hierarchy DIDs. (dictionary {'lpn': {key : value}}). Default=None
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+
+    """
+    scopes = list_scopes(vo=vo, session=session)
+    dids = []
+    rses = {}
+    for lfn in lfns:
+        scope, name = extract_scope(lfn['lfn'], scopes)
+        dids.append({'scope': scope, 'name': name})
+        rse = lfn['rse']
+        if rse not in rses:
+            rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+            rses[rse] = rse_id
+        lfn['rse_id'] = rses[rse]
+
+    # Check if the issuer can add dids and use skip_availabitlity
+    for rse in rses:
+        rse_id = rses[rse]
+        kwargs = {'rse': rse, 'rse_id': rse_id}
+        auth_result = has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo, session=session)
+        if not auth_result.allowed:
+            raise AccessDenied('Account %s can not add file replicas on %s for VO %s. %s' % (issuer, rse, vo, auth_result.message))
+        if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs, vo=vo, session=session):
+            ignore_availability = False
+
+    # Check if the issuer can add the files
+    kwargs = {'issuer': issuer, 'dids': dids}
+    auth_result = has_permission(issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo, session=session)
+    if not auth_result.allowed:
+        raise AccessDenied('Account %s can not bulk add data identifier for VO %s. %s' % (issuer, vo, auth_result.message))
+
+    dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, parents_metadata=parents_metadata, vo=vo, session=session)

rucio/gateway/exporter.py

@@ -0,0 +1,60 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any
+
+from rucio.common import exception
+from rucio.core import exporter
+from rucio.core.rse import get_rse_name
+from rucio.db.sqla.session import read_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def export_data(issuer: str, distance: bool = True, vo: str = 'def', *, session: "Session") -> dict[str, Any]:
+    """
+    Export data from Rucio.
+
+    :param issuer: the issuer.
+    :param distance: To enable the reporting of distance.
+    :param vo: the VO of the issuer.
+    :param session: The database session in use.
+    """
+    kwargs = {'issuer': issuer}
+    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='export', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account %s can not export data. %s' % (issuer, auth_result.message))
+
+    data = exporter.export_data(distance=distance, vo=vo, session=session)
+    rses = {}
+    distances = {}
+
+    for rse_id in data['rses']:
+        rse = data['rses'][rse_id]
+        rses[get_rse_name(rse_id=rse_id, session=session)] = rse
+    data['rses'] = rses
+
+    if distance:
+        for src_id in data['distances']:
+            dests = data['distances'][src_id]
+            src = get_rse_name(rse_id=src_id, session=session)
+            distances[src] = {}
+            for dest_id in dests:
+                dest = get_rse_name(rse_id=dest_id, session=session)
+                distances[src][dest] = dests[dest_id]
+        data['distances'] = distances
+    return data

rucio/gateway/heartbeat.py

@@ -0,0 +1,76 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Optional
+
+from rucio.common import exception
+from rucio.core import heartbeat
+from rucio.db.sqla.session import read_session, transactional_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from threading import Thread
+
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def list_heartbeats(issuer: Optional[str] = None, vo: str = 'def', *, session: "Session") -> list["heartbeat.HeartbeatDict"]:
+    """
+    Return a list of tuples of all heartbeats.
+
+    :param issuer: The issuer account.
+    :param vo: the VO for the issuer.
+    :param session: The database session in use.
+    :returns: List of tuples [('Executable', 'Hostname', ...), ...]
+    """
+
+    kwargs = {'issuer': issuer}
+    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='list_heartbeats', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('%s cannot list heartbeats. %s' % (issuer, auth_result.message))
+    return heartbeat.list_heartbeats(session=session)
+
+
+@transactional_session
+def create_heartbeat(
+    executable: str,
+    hostname: str,
+    pid: int,
+    older_than: int,
+    payload: Optional[str],
+    thread: Optional["Thread"] = None,
+    issuer: Optional[str] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Creates a heartbeat.
+    :param issuer: The issuer account.
+    :param vo: the VO for the issuer.
+    :param executable: Executable name as a string, e.g., conveyor-submitter.
+    :param hostname: Hostname as a string, e.g., rucio-daemon-prod-01.cern.ch.
+    :param pid: UNIX Process ID as a number, e.g., 1234.
+    :param thread: Python Thread Object.
+    :param older_than: Ignore specified heartbeats older than specified nr of seconds.
+    :param payload: Payload identifier which can be further used to identify the work a certain thread is executing.
+    :param session: The database session in use.
+
+    """
+    kwargs = {'issuer': issuer}
+    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='send_heartbeats', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('%s cannot send heartbeats. %s' % (issuer, auth_result.message))
+    heartbeat.live(executable=executable, hostname=hostname, pid=pid, thread=thread, older_than=older_than, payload=payload, session=session)

rucio/gateway/identity.py

@@ -0,0 +1,189 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Interface for identity abstraction layer
+"""
+
+from typing import TYPE_CHECKING, Optional
+
+from rucio.common import exception
+from rucio.common.types import InternalAccount
+from rucio.core import identity
+from rucio.db.sqla.constants import DatabaseOperationType, IdentityType
+from rucio.db.sqla.session import db_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sqlalchemy import Row
+
+
+def add_identity(
+    identity_key: str,
+    id_type: str,
+    email: str,
+    password: Optional[str] = None,
+) -> None:
+    """
+    Creates a user identity.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml)
+    :param email: The Email address associated with the identity.
+    :param password: If type==userpass, this sets the password.
+    """
+    with db_session(DatabaseOperationType.WRITE) as session:
+        return identity.add_identity(identity_key, IdentityType[id_type.upper()], email, password=password, session=session)
+
+
+def del_identity(
+    identity_key: str,
+    id_type: str,
+    issuer: str,
+    vo: str = 'def',
+) -> None:
+    """
+    Deletes a user identity.
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param issuer: The issuer account.
+    :param vo: the VO of the issuer.
+    """
+    converted_id_type = IdentityType[id_type.upper()]
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        kwargs = {'accounts': identity.list_accounts_for_identity(identity_key, converted_id_type, session=session)}
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='del_identity', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not delete identity. %s' % (issuer, auth_result.message))
+
+        return identity.del_identity(identity_key, converted_id_type, session=session)
+
+
+def add_account_identity(
+    identity_key: str,
+    id_type: str,
+    account: str,
+    email: str,
+    issuer: str,
+    default: bool = False,
+    password: Optional[str] = None,
+    vo: str = 'def',
+) -> None:
+    """
+    Adds a membership association between identity and account.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param account: The account name.
+    :param email: The Email address associated with the identity.
+    :param issuer: The issuer account.
+    :param default: If True, the account should be used by default with the provided identity.
+    :param password: Password if id_type is userpass.
+    :param vo: the VO to act on.
+    """
+    kwargs = {'identity': identity_key, 'type': id_type, 'account': account}
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_account_identity', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not add account identity. %s' % (issuer, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
+
+        return identity.add_account_identity(identity=identity_key, type_=IdentityType[id_type.upper()], default=default,
+                                             email=email, account=internal_account, password=password, session=session)
+
+
+def verify_identity(identity_key: str, id_type: str, password: Optional[str] = None) -> bool:
+    """
+    Verifies a user identity.
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml)
+    :param password: If type==userpass, verifies the identity_key, .
+    """
+    with db_session(DatabaseOperationType.READ) as session:
+        return identity.verify_identity(identity_key, IdentityType[id_type.upper()], password=password, session=session)
+
+
+def del_account_identity(
+    identity_key: str,
+    id_type: str,
+    account: str,
+    issuer: str,
+    vo: str = 'def',
+) -> None:
+    """
+    Removes a membership association between identity and account.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param account: The account name.
+    :param issuer: The issuer account.
+    :param vo: the VO to act on.
+    """
+    kwargs = {'account': account}
+
+    with db_session(DatabaseOperationType.WRITE) as session:
+        auth_result = permission.has_permission(issuer=issuer, vo=vo, action='del_account_identity', kwargs=kwargs, session=session)
+        if not auth_result.allowed:
+            raise exception.AccessDenied('Account %s can not delete account identity. %s' % (issuer, auth_result.message))
+
+        internal_account = InternalAccount(account, vo=vo)
+
+        return identity.del_account_identity(identity_key, IdentityType[id_type.upper()], internal_account, session=session)
+
+
+def list_identities(**kwargs) -> "Sequence[Row[tuple[str, IdentityType]]]":
+    """
+    Returns a list of all enabled identities.
+
+    returns: A list of all enabled identities.
+    """
+    with db_session(DatabaseOperationType.READ) as session:
+        return identity.list_identities(session=session, **kwargs)
+
+
+def get_default_account(
+    identity_key: str,
+    id_type: str,
+) -> str:
+    """
+    Returns the default account for this identity.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    """
+    with db_session(DatabaseOperationType.READ) as session:
+        account = identity.get_default_account(identity_key, IdentityType[id_type.upper()], session=session)
+    return account.external
+
+
+def list_accounts_for_identity(
+    identity_key: str,
+    id_type: str,
+) -> list[str]:
+    """
+    Returns a list of all accounts for an identity.
+
+    :param identity: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+
+    returns: A list of all accounts for the identity.
+    """
+    with db_session(DatabaseOperationType.READ) as session:
+        accounts = identity.list_accounts_for_identity(identity_key, IdentityType[id_type.upper()], session=session)
+    return [account.external for account in accounts]

rucio/gateway/importer.py

@@ -0,0 +1,46 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any
+
+from rucio.common import exception
+from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount
+from rucio.core import importer
+from rucio.db.sqla.session import transactional_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def import_data(data: dict[str, Any], issuer: str, vo: str = 'def', *, session: "Session") -> None:
+    """
+    Import data to add/update/delete records in Rucio.
+
+    :param data: data to be imported.
+    :param issuer: the issuer.
+    :param vo: the VO of the issuer.
+    :param session: The database session in use.
+    """
+    kwargs = {'issuer': issuer}
+    validate_schema(name='import', obj=data, vo=vo)
+    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account %s can not import data. %s' % (issuer, auth_result.message))
+
+    for account in data.get('accounts', []):
+        account['account'] = InternalAccount(account['account'], vo=vo)
+    return importer.import_data(data, vo=vo, session=session)

rucio/gateway/lifetime_exception.py

@@ -0,0 +1,121 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common import exception
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.utils import gateway_update_return_dict
+from rucio.core import lifetime_exception
+from rucio.db.sqla.session import stream_session, transactional_session
+from rucio.gateway import permission
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Iterator
+
+    from sqlalchemy.orm import Session
+
+    from rucio.db.sqla.constants import LifetimeExceptionsState
+
+
+@stream_session
+def list_exceptions(
+    exception_id: Optional[str] = None,
+    states: Optional["Iterable[LifetimeExceptionsState]"] = None,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> 'Iterator[dict[str, Any]]':
+    """
+    List exceptions to Lifetime Model.
+
+    :param id:         The id of the exception
+    :param states:     The states to filter
+    :param vo:         The VO to act on
+    :param session:    The database session in use.
+    """
+
+    exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states, session=session)
+    for e in exceptions:
+        if vo == e['scope'].vo:
+            yield gateway_update_return_dict(e, session=session)
+
+
+@transactional_session
+def add_exception(
+    dids: "Iterable[dict[str, Any]]",
+    account: str,
+    pattern: str,
+    comments: str,
+    expires_at: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> dict[str, Any]:
+    """
+    Add exceptions to Lifetime Model.
+
+    :param dids:        The list of dids
+    :param account:     The account of the requester.
+    :param pattern:     The account.
+    :param comments:    The comments associated to the exception.
+    :param expires_at:  The expiration date of the exception.
+    :param vo:          The VO to act on.
+    :param session:     The database session in use.
+
+    returns:            The id of the exception.
+    """
+
+    internal_account = InternalAccount(account, vo=vo)
+    for did in dids:
+        did['scope'] = InternalScope(did['scope'], vo=vo)
+    exceptions = lifetime_exception.add_exception(dids=dids, account=internal_account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
+
+    for key in exceptions:
+        if key == 'exceptions':
+            for reqid in exceptions[key]:
+                for did in exceptions[key][reqid]:
+                    did['scope'] = did['scope'].external
+                    did['did_type'] = did['did_type'].name
+        else:
+            for did in exceptions[key]:
+                did['scope'] = did['scope'].external
+                did['did_type'] = did['did_type'].name
+
+    return exceptions
+
+
+@transactional_session
+def update_exception(
+    exception_id: str,
+    state: 'LifetimeExceptionsState',
+    issuer: str,
+    vo: str = 'def',
+    *,
+    session: "Session"
+) -> None:
+    """
+    Update exceptions state to Lifetime Model.
+
+    :param id:         The id of the exception.
+    :param state:      The states to filter.
+    :param issuer:     The issuer account.
+    :param vo:         The VO to act on.
+    :param session:    The database session in use.
+    """
+    kwargs = {'exception_id': exception_id, 'vo': vo}
+    auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_lifetime_exceptions', kwargs=kwargs, session=session)
+    if not auth_result.allowed:
+        raise exception.AccessDenied('Account %s can not update lifetime exceptions. %s' % (issuer, auth_result.message))
+    return lifetime_exception.update_exception(exception_id=exception_id, state=state, session=session)