rucio 37.0.0rc1__py3-none-any.whl

rucio/core/vo.py

@@ -0,0 +1,169 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import select, update
+from sqlalchemy.exc import DatabaseError, IntegrityError, NoResultFound
+
+from rucio.common import exception
+from rucio.common.config import config_get, config_get_bool
+from rucio.common.types import InternalAccount
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import AccountType, IdentityType
+from rucio.db.sqla.session import read_session, transactional_session
+
+# Format for long VO names
+LONG_VO_RE = re.compile(r"^[a-zA-Z0-9\.\-]+$")
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def vo_exists(vo: str, *, session: "Session") -> bool:
+    """
+    Verify that the vo exists.
+
+    :param vo: The vo to verify.
+    :param session: The db session in use.
+
+    :returns: True if the vo is in the vo table, False otherwise
+    """
+    stmt = select(
+        models.VO
+    ).where(
+        models.VO.vo == vo
+    )
+    return bool(session.execute(stmt).scalar())
+
+
+@transactional_session
+def add_vo(vo: str, description: str, email: str, *, session: "Session") -> None:
+    """
+    Add a VO and setup a new root user.
+    New root user will have account name 'root' and a userpass identity with username: 'root@<vo>' and password: 'password'
+
+    :param vo: 3-letter unique tag for a VO.
+    :param description: Descriptive string for the VO (e.g. Full name).
+    :param email: Contact email for the VO.
+    :param session: The db session in use.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    if len(vo) != 3:
+        raise exception.RucioException('Invalid VO tag, must be 3 chars.')
+
+    new_vo = models.VO(vo=vo, description=description, email=email)
+
+    try:
+        new_vo.save(session=session)
+    except IntegrityError:
+        raise exception.Duplicate('VO {} already exists!'.format(vo))
+    except DatabaseError as error:
+        raise exception.RucioException(error.args)
+
+    from rucio.core.account import add_account, list_identities
+    from rucio.core.identity import add_account_identity
+    new_root = InternalAccount('root', vo=vo)
+    add_account(account=new_root, type_=AccountType['SERVICE'], email=email, session=session)
+    add_account_identity(identity='root@{}'.format(vo),
+                         type_=IdentityType['USERPASS'],
+                         account=new_root,
+                         email=email,
+                         default=False,
+                         password='password',  # noqa: S106
+                         session=session)
+
+    for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session):
+        add_account_identity(identity=ident['identity'], type_=ident['type'], account=new_root, email='', session=session)
+
+
+@read_session
+def list_vos(*, session: "Session") -> list[dict[str, Any]]:
+    """
+    List all the VOs in the db.
+
+    :param session: The db session in use.
+    :returns: List of VO dictionaries.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    stmt = select(
+        models.VO
+    )
+
+    vos = []
+    for vo in session.execute(stmt).scalars().all():
+        vo_dict = {'vo': vo.vo,
+                   'description': vo.description,
+                   'email': vo.email,
+                   'created_at': vo.created_at,
+                   'updated_at': vo.updated_at}
+        vos.append(vo_dict)
+
+    return vos
+
+
+@transactional_session
+def update_vo(vo: str, parameters: dict[str, Any], *, session: "Session") -> None:
+    """
+    Update VO properties (email, description).
+
+    :param vo: The VO to update.
+    :param parameters: A dictionary with the new properties.
+    :param session: The db session in use.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    try:
+        stmt = select(
+            models.VO
+        ).where(
+            models.VO.vo == vo
+        )
+        session.execute(stmt).scalar_one()
+    except NoResultFound:
+        raise exception.VONotFound('VO {} not found'.format(vo))
+    param = {}
+    for key in parameters:
+        if key in ['email', 'description']:
+            param[key] = parameters[key]
+    stmt = update(
+        models.VO
+    ).where(
+        models.VO.vo == vo
+    ).values(
+        param
+    )
+    session.execute(stmt)
+
+
+def map_vo(vo: str) -> str:
+    """
+    Converts a long VO name into the internal short (three letter)
+    tag mapping.
+    Mappings are loaded from the vo-map section of the config database table.
+    If a mapping is not found, the original is returned unchanged.
+    :param vo: The long VO name string.
+    :returns: The short VO name string.
+    """
+    # Newline is ignored by regexp if at end of string, so test for that as well.
+    if not LONG_VO_RE.match(vo) or '\n' in vo:
+        raise exception.RucioException('Invalid characters in VO name.')
+    return config_get("vo-map", vo, raise_exception=False, default=vo)

rucio/core/volatile_replica.py

@@ -0,0 +1,151 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from sqlalchemy import and_, delete, exists, insert, or_, select, true, update
+from sqlalchemy.exc import NoResultFound
+
+from rucio.common import exception
+from rucio.core.rse import get_rse_name
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import ReplicaState
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_volatile_replicas(rse_id: str, replicas: "Iterable[dict[str, Any]]", *, session: "Session") -> None:
+    """
+    Bulk add volatile replicas.
+
+    :param rse_id: the rse id.
+    :param replicas: the iterable of volatile replicas.
+    :param session: The database session in use.
+    """
+    # first check that the rse is a volatile one
+    try:
+        stmt = select(
+            models.RSE
+        ).where(
+            and_(models.RSE.id == rse_id,
+                 models.RSE.volatile == true())
+        )
+        session.execute(stmt).one()
+    except NoResultFound:
+        raise exception.UnsupportedOperation('No volatile rse found for %s !'
+                                             % get_rse_name(rse_id=rse_id, session=session))
+
+    file_clause, replica_clause = [], []
+    for replica in replicas:
+        file_clause.append(and_(models.DataIdentifier.scope == replica['scope'],
+                                models.DataIdentifier.name == replica['name'],
+                                ~exists(select(1).prefix_with('/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'))
+                                .where(and_(models.RSEFileAssociation.scope == replica['scope'],
+                                            models.RSEFileAssociation.name == replica['name'],
+                                            models.RSEFileAssociation.rse_id == rse_id))))
+        replica_clause.append(and_(models.RSEFileAssociation.scope == replica['scope'],
+                                   models.RSEFileAssociation.name == replica['name'],
+                                   models.RSEFileAssociation.rse_id == rse_id))
+    if replica_clause:
+        now = datetime.utcnow()
+        stmt = update(
+            models.RSEFileAssociation
+        ).prefix_with(
+            '/*+ INDEX(REPLICAS REPLICAS_PK) */',
+            dialect='oracle'
+        ).where(
+            or_(*replica_clause)
+        ).execution_options(
+            synchronize_session=False
+        ).values({
+            models.RSEFileAssociation.updated_at: now,
+            models.RSEFileAssociation.tombstone: now
+        })
+        session.execute(stmt)
+
+    if file_clause:
+        stmt = select(
+            models.DataIdentifier.scope,
+            models.DataIdentifier.name,
+            models.DataIdentifier.bytes,
+            models.DataIdentifier.md5,
+            models.DataIdentifier.adler32
+        ).filter(
+            or_(*file_clause)
+        )
+
+        new_replicas = [
+            {
+                'rse_id': rse_id,
+                'adler32': adler32,
+                'state': ReplicaState.AVAILABLE,
+                'scope': scope,
+                'name': name,
+                'lock_cnt': 0,
+                'tombstone': datetime.utcnow(),
+                'bytes': bytes_,
+                'md5': md5
+            }
+            for scope, name, bytes_, md5, adler32 in session.execute(stmt).all()
+        ]
+        if new_replicas:
+            stmt = insert(
+                models.RSEFileAssociation
+            )
+            session.execute(stmt, new_replicas)
+
+
+@transactional_session
+def delete_volatile_replicas(rse_id: str, replicas: "Iterable[dict[str, Any]]", *, session: "Session") -> None:
+    """
+    Bulk delete volatile replicas.
+
+    :param rse_id: the rse id.
+    :param replicas: the iterable of volatile replicas.
+    :param session: The database session in use.
+    """
+    # first check that the rse is a volatile one
+    try:
+        stmt = select(
+            models.RSE
+        ).where(
+            and_(models.RSE.id == rse_id,
+                 models.RSE.volatile == true())
+        )
+        session.execute(stmt).one()
+    except NoResultFound:
+        raise exception.UnsupportedOperation('No volatile rse found for %s !'
+                                             % get_rse_name(rse_id=rse_id, session=session))
+
+    conditions = []
+    for replica in replicas:
+        conditions.append(and_(models.RSEFileAssociation.scope == replica['scope'],
+                               models.RSEFileAssociation.name == replica['name']))
+
+    if conditions:
+        stmt = delete(
+            models.RSEFileAssociation
+        ).where(
+            and_(models.RSEFileAssociation.rse_id == rse_id,
+                 or_(*conditions))
+        ).execution_options(
+            synchronize_session=False
+        )
+        session.execute(stmt)

rucio/daemons/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/abacus/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/abacus/account.py

@@ -0,0 +1,116 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-Account is a daemon to update Account counters.
+"""
+
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.common.utils import get_thread_with_periodic_running_function
+from rucio.core.account_counter import fill_account_counter_history_table, get_updated_account_counters, update_account_counter
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-account'
+
+
+def account_update(
+        once: bool = False,
+        sleep_time: int = 10
+) -> None:
+    """
+    Main loop to check and update the Account Counters.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=run_once,
+    )
+
+
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        **_kwargs
+) -> None:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    start = time.time()  # NOQA
+    updated_account_counters = get_updated_account_counters(total_workers=total_workers,
+                                                            worker_number=worker_number)
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(updated_account_counters)))
+
+    # If the list is empty, sent the worker to sleep
+    if not updated_account_counters:
+        logger(logging.INFO, 'did not get any work')
+        return
+
+    for account_counter in updated_account_counters:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_account_counter(account=account_counter['account'], rse_id=account_counter['rse_id'])
+        logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_counter['account'], account_counter['rse_id'], time.time() - start_time))
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(
+        once: bool = False,
+        threads: int = 1,
+        fill_history_table: bool = False,
+        sleep_time: int = 10
+) -> None:
+    """
+    Starts up the Abacus-Account threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        account_update(once)
+    else:
+        logging.info('main: starting threads')
+        thread_list = [threading.Thread(target=account_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                       range(0, threads)]
+        if fill_history_table:
+            thread_list.append(get_thread_with_periodic_running_function(3600, fill_account_counter_history_table, graceful_stop))
+        [t.start() for t in thread_list]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]

rucio/daemons/abacus/collection_replica.py

@@ -0,0 +1,124 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-Collection-Replica is a daemon to update collection replica.
+"""
+import functools
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.core.replica import get_cleaned_updated_collection_replicas, update_collection_replica
+from rucio.daemons.common import HeartbeatHandler, run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-collection-replica'
+
+
+def collection_replica_update(
+        once: bool = False,
+        limit: int = 1000,
+        sleep_time: int = 10
+) -> None:
+    """
+    Main loop to check and update the collection replicas.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            limit=limit,
+        ),
+    )
+
+
+def run_once(
+        heartbeat_handler: HeartbeatHandler,
+        limit: int,
+        **_kwargs
+) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    # Select a bunch of collection replicas for to update for this worker
+    start = time.time()  # NOQA
+    replicas = get_cleaned_updated_collection_replicas(total_workers=total_workers - 1,
+                                                       worker_number=worker_number,
+                                                       limit=limit)
+
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(replicas)))
+    # If the list is empty, sent the worker to sleep
+    if not replicas:
+        logger(logging.INFO, 'did not get any work')
+        must_sleep = True
+        return must_sleep
+
+    for replica in replicas:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_collection_replica(replica)
+        logger(logging.DEBUG, 'update of collection replica "%s" took %f' % (replica['id'], time.time() - start_time))
+
+    must_sleep = False
+    if limit and len(replicas) < limit:
+        must_sleep = True
+    return must_sleep
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(
+        once: bool = False,
+        threads: int = 1,
+        sleep_time: int = 10,
+        limit: int = 1000):
+    """
+    Starts up the Abacus-Collection-Replica threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        collection_replica_update(once)
+    else:
+        logging.info('main: starting threads')
+        thread_list = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time, 'limit': limit})
+                       for _ in range(0, threads)]
+        [t.start() for t in thread_list]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while thread_list[0].is_alive():
+            [t.join(timeout=3.14) for t in thread_list]