rucio 32.8.6__py3-none-any.whl

rucio/api/dirac.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.api.permission import has_permission
+from rucio.api.scope import list_scopes
+from rucio.common.exception import AccessDenied
+from rucio.common.utils import extract_scope
+from rucio.core import dirac
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_files(lfns, issuer, ignore_availability, vo='def', *, session: "Session"):
+    """
+    Bulk add files :
+    - Create the file and replica.
+    - If doesn't exist create the dataset containing the file as well as a rule on the dataset on ANY sites.
+    - Create all the ascendants of the dataset if they do not exist
+
+    :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
+    :param issuer: The issuer account.
+    :param ignore_availability: A boolean to ignore blocked sites.
+    :param vo: The VO to act on.
+    :param session: The database session in use.
+
+    """
+    scopes = list_scopes(vo=vo, session=session)
+    dids = []
+    rses = {}
+    for lfn in lfns:
+        scope, name = extract_scope(lfn['lfn'], scopes)
+        dids.append({'scope': scope, 'name': name})
+        rse = lfn['rse']
+        if rse not in rses:
+            rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+            rses[rse] = rse_id
+        lfn['rse_id'] = rses[rse]
+
+    # Check if the issuer can add dids and use skip_availabitlity
+    for rse in rses:
+        rse_id = rses[rse]
+        kwargs = {'rse': rse, 'rse_id': rse_id}
+        if not has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs, vo=vo, session=session):
+            raise AccessDenied('Account %s can not add file replicas on %s for VO %s' % (issuer, rse, vo))
+        if not has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs, vo=vo, session=session):
+            ignore_availability = False
+
+    # Check if the issuer can add the files
+    kwargs = {'issuer': issuer, 'dids': dids}
+    if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo, session=session):
+        raise AccessDenied('Account %s can not bulk add data identifier for VO %s' % (issuer, vo))
+
+    dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, vo=vo, session=session)

rucio/api/exporter.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.api import permission
+from rucio.common import exception
+from rucio.core import exporter
+from rucio.core.rse import get_rse_name
+from rucio.db.sqla.session import read_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def export_data(issuer, distance=True, vo='def', *, session: "Session"):
+    """
+    Export data from Rucio.
+
+    :param issuer: the issuer.
+    :param distance: To enable the reporting of distance.
+    :param vo: the VO of the issuer.
+    :param session: The database session in use.
+    """
+    kwargs = {'issuer': issuer}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='export', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not export data' % issuer)
+
+    data = exporter.export_data(distance=distance, vo=vo, session=session)
+    rses = {}
+    distances = {}
+
+    for rse_id in data['rses']:
+        rse = data['rses'][rse_id]
+        rses[get_rse_name(rse_id=rse_id, session=session)] = rse
+    data['rses'] = rses
+
+    if distance:
+        for src_id in data['distances']:
+            dests = data['distances'][src_id]
+            src = get_rse_name(rse_id=src_id, session=session)
+            distances[src] = {}
+            for dest_id in dests:
+                dest = get_rse_name(rse_id=dest_id, session=session)
+                distances[src][dest] = dests[dest_id]
+        data['distances'] = distances
+    return data

rucio/api/heartbeat.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.api import permission
+from rucio.common import exception
+from rucio.core import heartbeat
+from rucio.db.sqla.session import read_session, transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def list_heartbeats(issuer=None, vo='def', *, session: "Session"):
+    """
+    Return a list of tuples of all heartbeats.
+
+    :param issuer: The issuer account.
+    :param vo: the VO for the issuer.
+    :param session: The database session in use.
+    :returns: List of tuples [('Executable', 'Hostname', ...), ...]
+    """
+
+    kwargs = {'issuer': issuer}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='list_heartbeats', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('%s cannot list heartbeats' % issuer)
+    return heartbeat.list_heartbeats(session=session)
+
+
+@transactional_session
+def create_heartbeat(executable, hostname, pid, older_than, payload, thread=None, issuer=None, vo='def', *, session: "Session"):
+    """
+    Creates a heartbeat.
+    :param issuer: The issuer account.
+    :param vo: the VO for the issuer.
+    :param executable: Executable name as a string, e.g., conveyor-submitter.
+    :param hostname: Hostname as a string, e.g., rucio-daemon-prod-01.cern.ch.
+    :param pid: UNIX Process ID as a number, e.g., 1234.
+    :param thread: Python Thread Object.
+    :param older_than: Ignore specified heartbeats older than specified nr of seconds.
+    :param payload: Payload identifier which can be further used to identify the work a certain thread is executing.
+    :param session: The database session in use.
+
+    """
+    kwargs = {'issuer': issuer}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='send_heartbeats', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('%s cannot send heartbeats' % issuer)
+    heartbeat.live(executable=executable, hostname=hostname, pid=pid, thread=thread, older_than=older_than, payload=payload, session=session)

rucio/api/identity.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Interface for identity abstraction layer
+"""
+
+from typing import TYPE_CHECKING
+
+from rucio.api import permission
+from rucio.common import exception
+from rucio.common.types import InternalAccount
+from rucio.core import identity
+from rucio.db.sqla.constants import IdentityType
+from rucio.db.sqla.session import read_session, transactional_session
+
+if TYPE_CHECKING:
+    from typing import Optional
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_identity(identity_key, id_type, email, password=None, *, session: "Session"):
+    """
+    Creates a user identity.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml)
+    :param email: The Email address associated with the identity.
+    :param password: If type==userpass, this sets the password.
+    :param session: The database session in use.
+    """
+    return identity.add_identity(identity_key, IdentityType[id_type.upper()], email, password=password, session=session)
+
+
+@transactional_session
+def del_identity(identity_key, id_type, issuer, vo='def', *, session: "Session"):
+    """
+    Deletes a user identity.
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param issuer: The issuer account.
+    :param vo: the VO of the issuer.
+    :param session: The database session in use.
+    """
+    id_type = IdentityType[id_type.upper()]
+    kwargs = {'accounts': identity.list_accounts_for_identity(identity_key, id_type, session=session)}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='del_identity', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not delete identity' % (issuer))
+
+    return identity.del_identity(identity_key, id_type, session=session)
+
+
+@transactional_session
+def add_account_identity(identity_key, id_type, account, email, issuer, default=False, password=None, vo='def', *, session: "Session"):
+    """
+    Adds a membership association between identity and account.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param account: The account name.
+    :param email: The Email address associated with the identity.
+    :param issuer: The issuer account.
+    :param default: If True, the account should be used by default with the provided identity.
+    :param password: Password if id_type is userpass.
+    :param vo: the VO to act on.
+    :param session: The database session in use.
+    """
+    kwargs = {'identity': identity_key, 'type': id_type, 'account': account}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='add_account_identity', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not add account identity' % (issuer))
+
+    account = InternalAccount(account, vo=vo)
+
+    return identity.add_account_identity(identity=identity_key, type_=IdentityType[id_type.upper()], default=default,
+                                         email=email, account=account, password=password, session=session)
+
+
+@read_session
+def verify_identity(identity_key: str, id_type: str, password: "Optional[str]" = None, *, session: "Session") -> bool:
+    """
+    Verifies a user identity.
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml)
+    :param password: If type==userpass, verifies the identity_key, .
+    :param session: The database session in use.
+    """
+    return identity.verify_identity(identity_key, IdentityType[id_type.upper()], password=password, session=session)
+
+
+@transactional_session
+def del_account_identity(identity_key, id_type, account, issuer, vo='def', *, session: "Session"):
+    """
+    Removes a membership association between identity and account.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param account: The account name.
+    :param issuer: The issuer account.
+    :param vo: the VO to act on.
+    :param session: The database session in use.
+    """
+    kwargs = {'account': account}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='del_account_identity', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not delete account identity' % (issuer))
+
+    account = InternalAccount(account, vo=vo)
+
+    return identity.del_account_identity(identity_key, IdentityType[id_type.upper()], account, session=session)
+
+
+@read_session
+def list_identities(*, session: "Session", **kwargs):
+    """
+    Returns a list of all enabled identities.
+
+    :param session: The database session in use.
+    returns: A list of all enabled identities.
+    """
+    return identity.list_identities(session=session, **kwargs)
+
+
+@read_session
+def get_default_account(identity_key, id_type, *, session: "Session"):
+    """
+    Returns the default account for this identity.
+
+    :param identity_key: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param session: The database session in use.
+    """
+    account = identity.get_default_account(identity_key, IdentityType[id_type.upper()], session=session)
+    return account.external
+
+
+@read_session
+def list_accounts_for_identity(identity_key, id_type, *, session: "Session"):
+    """
+    Returns a list of all accounts for an identity.
+
+    :param identity: The identity key name. For example x509 DN, or a username.
+    :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
+    :param session: The database session in use.
+
+    returns: A list of all accounts for the identity.
+    """
+    accounts = identity.list_accounts_for_identity(identity_key, IdentityType[id_type.upper()], session=session)
+    return [account.external for account in accounts]

rucio/api/importer.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.api import permission
+from rucio.common import exception
+from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount
+from rucio.core import importer
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def import_data(data, issuer, vo='def', *, session: "Session"):
+    """
+    Import data to add/update/delete records in Rucio.
+
+    :param data: data to be imported.
+    :param issuer: the issuer.
+    :param vo: the VO of the issuer.
+    :param session: The database session in use.
+    """
+    kwargs = {'issuer': issuer}
+    validate_schema(name='import', obj=data, vo=vo)
+    if not permission.has_permission(issuer=issuer, vo=vo, action='import', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not import data' % issuer)
+
+    for account in data.get('accounts', []):
+        account['account'] = InternalAccount(account['account'], vo=vo)
+    return importer.import_data(data, vo=vo, session=session)

rucio/api/lifetime_exception.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from rucio.api import permission
+from rucio.common import exception
+from rucio.common.types import InternalAccount, InternalScope
+from rucio.common.utils import api_update_return_dict
+from rucio.core import lifetime_exception
+from rucio.db.sqla.session import stream_session, transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@stream_session
+def list_exceptions(exception_id=None, states=None, vo='def', *, session: "Session"):
+    """
+    List exceptions to Lifetime Model.
+
+    :param id:         The id of the exception
+    :param states:     The states to filter
+    :param vo:         The VO to act on
+    :param session:    The database session in use.
+    """
+
+    exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states, session=session)
+    for e in exceptions:
+        if vo == e['scope'].vo:
+            yield api_update_return_dict(e, session=session)
+
+
+@transactional_session
+def add_exception(dids, account, pattern, comments, expires_at, vo='def', *, session: "Session"):
+    """
+    Add exceptions to Lifetime Model.
+
+    :param dids:        The list of dids
+    :param account:     The account of the requester.
+    :param pattern:     The account.
+    :param comments:    The comments associated to the exception.
+    :param expires_at:  The expiration date of the exception.
+    :param vo:          The VO to act on.
+    :param session:     The database session in use.
+
+    returns:            The id of the exception.
+    """
+
+    account = InternalAccount(account, vo=vo)
+    for did in dids:
+        did['scope'] = InternalScope(did['scope'], vo=vo)
+    exceptions = lifetime_exception.add_exception(dids=dids, account=account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
+
+    for key in exceptions:
+        if key == 'exceptions':
+            for reqid in exceptions[key]:
+                for did in exceptions[key][reqid]:
+                    did['scope'] = did['scope'].external
+                    did['did_type'] = did['did_type'].name
+        else:
+            for did in exceptions[key]:
+                did['scope'] = did['scope'].external
+                did['did_type'] = did['did_type'].name
+
+    return exceptions
+
+
+@transactional_session
+def update_exception(exception_id, state, issuer, vo='def', *, session: "Session"):
+    """
+    Update exceptions state to Lifetime Model.
+
+    :param id:         The id of the exception.
+    :param state:      The states to filter.
+    :param issuer:     The issuer account.
+    :param vo:         The VO to act on.
+    :param session:    The database session in use.
+    """
+    kwargs = {'exception_id': exception_id, 'vo': vo}
+    if not permission.has_permission(issuer=issuer, vo=vo, action='update_lifetime_exceptions', kwargs=kwargs, session=session):
+        raise exception.AccessDenied('Account %s can not update lifetime exceptions' % (issuer))
+    return lifetime_exception.update_exception(exception_id=exception_id, state=state, session=session)

rucio/api/lock.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import TYPE_CHECKING
+
+from rucio.common.types import InternalScope
+from rucio.common.utils import api_update_return_dict
+from rucio.core import lock
+from rucio.core.rse import get_rse_id
+from rucio.db.sqla.constants import DIDType
+from rucio.db.sqla.session import stream_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+LOGGER = logging.getLogger('lock')
+LOGGER.setLevel(logging.DEBUG)
+
+
+@stream_session
+def get_dataset_locks(scope, name, vo='def', *, session: "Session"):
+    """
+    Get the dataset locks of a dataset.
+
+    :param scope:          Scope of the dataset.
+    :param name:           Name of the dataset.
+    :param vo:             The VO to act on.
+    :param session:        The database session in use.
+    :return:               List of dicts {'rse_id': ..., 'state': ...}
+    """
+
+    scope = InternalScope(scope, vo=vo)
+
+    locks = lock.get_dataset_locks(scope=scope, name=name, session=session)
+
+    for lock_object in locks:
+        yield api_update_return_dict(lock_object, session=session)
+
+
+@stream_session
+def get_dataset_locks_bulk(dids, vo='def', *, session: "Session"):
+    """
+    Get the dataset locks for multiple datasets or containers.
+
+    :param dids:            List of dataset or container DIDs as dictionaries {"scope":..., "name":..., "type":...}
+                            "type" is optional. If present, will be either DIDType.DATASET or DIDType.CONTAINER,
+                            or string "dataset" or "container"
+    :param vo:              The VO to act on.
+    :param session:         The database session in use.
+    :return:                Generator of dicts describing found locks {'rse_id': ..., 'state': ...}. Duplicates are removed
+    """
+
+    if vo is None:
+        vo = "def"
+
+    dids_converted = []
+    for did_in in dids:
+        did = did_in.copy()
+        if isinstance(did.get("type"), str):
+            # convert DID type
+            try:
+                did["type"] = {
+                    "dataset": DIDType.DATASET,
+                    "container": DIDType.CONTAINER
+                }[did["type"]]
+            except KeyError:
+                raise ValueError("Unknown DID type %(type)s" % did)
+        if isinstance(did["scope"], str):
+            did["scope"] = InternalScope(did["scope"], vo=vo)
+        dids_converted.append(did)
+
+    seen = set()
+    for lock_info in lock.get_dataset_locks_bulk(dids_converted, session=session):
+        # filter duplicates - same scope, name, rse_id, rule_id
+        scope_str = str(lock_info["scope"])
+        key = (scope_str, lock_info["name"], lock_info["rse_id"], lock_info["rule_id"])
+        if key not in seen:
+            seen.add(key)
+            yield lock_info
+
+
+@stream_session
+def get_dataset_locks_by_rse(rse, vo='def', *, session: "Session"):
+    """
+    Get the dataset locks of an RSE.
+
+    :param rse:            RSE name.
+    :param vo:             The VO to act on.
+    :param session:        The database session in use.
+    :return:               List of dicts {'rse_id': ..., 'state': ...}
+    """
+
+    rse_id = get_rse_id(rse=rse, vo=vo, session=session)
+    locks = lock.get_dataset_locks_by_rse_id(rse_id=rse_id, session=session)
+
+    for lock_object in locks:
+        yield api_update_return_dict(lock_object, session=session)
+
+
+@stream_session
+def get_replica_locks_for_rule_id(rule_id, vo='def', *, session: "Session"):
+    """
+    Get the replica locks for a rule_id.
+
+    :param rule_id:     Rule ID.
+    :param vo:          The VO to act on.
+    :param session:     The database session in use.
+    :return:            List of dicts.
+    """
+
+    locks = lock.get_replica_locks_for_rule_id(rule_id=rule_id, session=session)
+
+    for lock_object in locks:
+        if lock_object['scope'].vo != vo:  # rule is on a different VO, so don't return any locks
+            LOGGER.debug('rule id %s is not present on VO %s' % (rule_id, vo))
+            break
+        yield api_update_return_dict(lock_object, session=session)