rucio 32.8.6__py3-none-any.whl

rucio/core/vo.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import DatabaseError, IntegrityError
+from sqlalchemy.orm.exc import NoResultFound
+
+from rucio.common import exception
+from rucio.common.config import config_get_bool, config_get
+from rucio.common.types import InternalAccount
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import AccountType, IdentityType
+from rucio.db.sqla.session import read_session, transactional_session
+
+# Format for long VO names
+LONG_VO_RE = re.compile(r"^[a-zA-Z0-9\.\-]+$")
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@read_session
+def vo_exists(vo, *, session: "Session"):
+    """
+    Verify that the vo exists.
+
+    :param vo: The vo to verify.
+    :param session: The db session in use.
+
+    :returns: True if the vo is in the vo table, False otherwise
+    """
+    return True if session.query(models.VO).filter_by(vo=vo).first() else False
+
+
+@transactional_session
+def add_vo(vo, description, email, *, session: "Session"):
+    """
+    Add a VO and setup a new root user.
+    New root user will have account name 'root' and a userpass identity with username: 'root@<vo>' and password: 'password'
+
+    :param vo: 3-letter unique tag for a VO.
+    :param descrition: Descriptive string for the VO (e.g. Full name).
+    :param email: Contact email for the VO.
+    :param session: The db session in use.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    if len(vo) != 3:
+        raise exception.RucioException('Invalid VO tag, must be 3 chars.')
+
+    new_vo = models.VO(vo=vo, description=description, email=email)
+
+    try:
+        new_vo.save(session=session)
+    except IntegrityError:
+        raise exception.Duplicate('VO {} already exists!'.format(vo))
+    except DatabaseError as error:
+        raise exception.RucioException(error.args)
+
+    from rucio.core.account import add_account, list_identities
+    from rucio.core.identity import add_account_identity
+    new_root = InternalAccount('root', vo=vo)
+    add_account(account=new_root, type_=AccountType['SERVICE'], email=email, session=session)
+    add_account_identity(identity='root@{}'.format(vo),
+                         type_=IdentityType['USERPASS'],
+                         account=new_root,
+                         email=email,
+                         default=False,
+                         password='password',
+                         session=session)
+
+    for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session):
+        add_account_identity(identity=ident['identity'], type_=ident['type'], account=new_root, email='', session=session)
+
+
+@read_session
+def list_vos(*, session: "Session"):
+    """
+    List all the VOs in the db.
+
+    :param session: The db session in use.
+    :returns: List of VO dictionaries.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    query = session.query(models.VO)
+
+    vos = []
+    for vo in query.all():
+        vo_dict = {'vo': vo.vo,
+                   'description': vo.description,
+                   'email': vo.email,
+                   'created_at': vo.created_at,
+                   'updated_at': vo.updated_at}
+        vos.append(vo_dict)
+
+    return vos
+
+
+@transactional_session
+def update_vo(vo, parameters, *, session: "Session"):
+    """
+    Update VO properties (email, description).
+
+    :param vo: The VO to update.
+    :param parameters: A dictionary with the new properties.
+    :param session: The db session in use.
+    """
+    if not config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
+        raise exception.UnsupportedOperation('VO operations cannot be performed in single VO mode.')
+
+    try:
+        query = session.query(models.VO).filter_by(vo=vo).one()
+    except NoResultFound:
+        raise exception.VONotFound('VO {} not found'.format(vo))
+    param = {}
+    for key in parameters:
+        if key in ['email', 'description']:
+            param[key] = parameters[key]
+    query.update(param)
+
+
+def map_vo(vo):
+    """
+    Converts a long VO name into the internal short (three letter)
+    tag mapping.
+    Mappings are loaded from the vo-map section of the config database table.
+    If a mapping is not found, the orignal is returned unchanged.
+    :param vo: The long VO name string.
+    :returns: The short VO name string.
+    """
+    # Newline is ignored by regexp if at end of string, so test for that as well.
+    if not LONG_VO_RE.match(vo) or '\n' in vo:
+        raise exception.RucioException('Invalid characters in VO name.')
+    return config_get("vo-map", vo, raise_exception=False, default=vo)

rucio/core/volatile_replica.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from sqlalchemy import and_, or_, exists, update, insert
+from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.sql.expression import select
+
+from rucio.common import exception
+from rucio.core.rse import get_rse_name
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import ReplicaState
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_volatile_replicas(rse_id, replicas, *, session: "Session"):
+    """
+    Bulk add volatile replicas.
+
+    :param rse_id: the rse id.
+    :param replicas: the list of volatile replicas.
+    :param session: The database session in use.
+    :returns: True is successful.
+    """
+    # first check that the rse is a volatile one
+    try:
+        session.query(models.RSE).filter_by(id=rse_id, volatile=True).one()
+    except NoResultFound:
+        raise exception.UnsupportedOperation('No volatile rse found for %s !'
+                                             % get_rse_name(rse_id=rse_id, session=session))
+
+    file_clause, replica_clause = [], []
+    for replica in replicas:
+        file_clause.append(and_(models.DataIdentifier.scope == replica['scope'],
+                                models.DataIdentifier.name == replica['name'],
+                                ~exists(select(1).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle'))
+                                .where(and_(models.RSEFileAssociation.scope == replica['scope'],
+                                            models.RSEFileAssociation.name == replica['name'],
+                                            models.RSEFileAssociation.rse_id == rse_id))))
+        replica_clause.append(and_(models.RSEFileAssociation.scope == replica['scope'],
+                                   models.RSEFileAssociation.name == replica['name'],
+                                   models.RSEFileAssociation.rse_id == rse_id))
+    if replica_clause:
+        now = datetime.utcnow()
+        stmt = update(models.RSEFileAssociation).\
+            prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle').\
+            where(or_(*replica_clause)).\
+            execution_options(synchronize_session=False).\
+            values(updated_at=now, tombstone=now)
+        session.execute(stmt)
+
+    if file_clause:
+        file_query = session.query(models.DataIdentifier.scope,
+                                   models.DataIdentifier.name,
+                                   models.DataIdentifier.bytes,
+                                   models.DataIdentifier.md5,
+                                   models.DataIdentifier.adler32).\
+            filter(or_(*file_clause))
+
+        new_replicas = [
+            {
+                'rse_id': rse_id,
+                'adler32': adler32,
+                'state': ReplicaState.AVAILABLE,
+                'scope': scope,
+                'name': name,
+                'lock_cnt': 0,
+                'tombstone': datetime.utcnow(),
+                'bytes': bytes_,
+                'md5': md5
+            }
+            for scope, name, bytes_, md5, adler32 in file_query
+        ]
+        if new_replicas:
+            session.execute(insert(models.RSEFileAssociation), new_replicas)
+
+
+@transactional_session
+def delete_volatile_replicas(rse_id, replicas, *, session: "Session"):
+    """
+    Bulk delete volatile replicas.
+
+    :param rse_id: the rse id.
+    :param replicas: the list of volatile replicas.
+    :param session: The database session in use.
+    :returns: True is successful.
+    """
+    # first check that the rse is a volatile one
+    try:
+        session.query(models.RSE).filter_by(id=rse_id, volatile=True).one()
+    except NoResultFound:
+        raise exception.UnsupportedOperation('No volatile rse found for %s !'
+                                             % get_rse_name(rse_id=rse_id, session=session))
+
+    conditions = []
+    for replica in replicas:
+        conditions.append(and_(models.RSEFileAssociation.scope == replica['scope'],
+                               models.RSEFileAssociation.name == replica['name']))
+
+    if conditions:
+        session.query(models.RSEFileAssociation).\
+            filter(models.RSEFileAssociation.rse_id == rse_id).\
+            filter(or_(*conditions)).\
+            delete(synchronize_session=False)

rucio/daemons/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/abacus/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/abacus/account.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-Account is a daemon to update Account counters.
+"""
+
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.common.utils import get_thread_with_periodic_running_function
+from rucio.core.account_counter import get_updated_account_counters, update_account_counter, fill_account_counter_history_table
+from rucio.daemons.common import run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-account'
+
+
+def account_update(once=False, sleep_time=10):
+    """
+    Main loop to check and update the Account Counters.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=run_once,
+    )
+
+
+def run_once(heartbeat_handler, **_kwargs):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    start = time.time()  # NOQA
+    account_rse_ids = get_updated_account_counters(total_workers=total_workers,
+                                                   worker_number=worker_number)
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(account_rse_ids)))
+
+    # If the list is empty, sent the worker to sleep
+    if not account_rse_ids:
+        logger(logging.INFO, 'did not get any work')
+        return
+
+    for account_rse_id in account_rse_ids:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_account_counter(account=account_rse_id[0], rse_id=account_rse_id[1])
+        logger(logging.DEBUG, 'update of account-rse counter "%s-%s" took %f' % (account_rse_id[0], account_rse_id[1], time.time() - start_time))
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
+    """
+    Starts up the Abacus-Account threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        account_update(once)
+    else:
+        logging.info('main: starting threads')
+        threads = [threading.Thread(target=account_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                   range(0, threads)]
+        if fill_history_table:
+            threads.append(get_thread_with_periodic_running_function(3600, fill_account_counter_history_table, graceful_stop))
+        [t.start() for t in threads]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while threads[0].is_alive():
+            [t.join(timeout=3.14) for t in threads]

rucio/daemons/abacus/collection_replica.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-Collection-Replica is a daemon to update collection replica.
+"""
+import functools
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.core.replica import get_cleaned_updated_collection_replicas, update_collection_replica
+from rucio.daemons.common import run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-collection-replica'
+
+
+def collection_replica_update(once=False, limit=1000, sleep_time=10):
+    """
+    Main loop to check and update the collection replicas.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            limit=limit,
+        ),
+    )
+
+
+def run_once(heartbeat_handler, limit, **_kwargs):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    # Select a bunch of collection replicas for to update for this worker
+    start = time.time()  # NOQA
+    replicas = get_cleaned_updated_collection_replicas(total_workers=total_workers - 1,
+                                                       worker_number=worker_number,
+                                                       limit=limit)
+
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(replicas)))
+    # If the list is empty, sent the worker to sleep
+    if not replicas:
+        logger(logging.INFO, 'did not get any work')
+        must_sleep = True
+        return must_sleep
+
+    for replica in replicas:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_collection_replica(replica)
+        logger(logging.DEBUG, 'update of collection replica "%s" took %f' % (replica['id'], time.time() - start_time))
+
+    must_sleep = False
+    if limit and len(replicas) < limit:
+        must_sleep = True
+    return must_sleep
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(once=False, threads=1, sleep_time=10, limit=1000):
+    """
+    Starts up the Abacus-Collection-Replica threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        collection_replica_update(once)
+    else:
+        logging.info('main: starting threads')
+        threads = [threading.Thread(target=collection_replica_update, kwargs={'once': once, 'sleep_time': sleep_time, 'limit': limit})
+                   for _ in range(0, threads)]
+        [t.start() for t in threads]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while threads[0].is_alive():
+            [t.join(timeout=3.14) for t in threads]

rucio/daemons/abacus/rse.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus-RSE is a daemon to update RSE counters.
+"""
+
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import rucio.db.sqla.util
+from rucio.common import exception
+from rucio.common.logging import setup_logging
+from rucio.common.utils import get_thread_with_periodic_running_function
+from rucio.core.rse_counter import get_updated_rse_counters, update_rse_counter, fill_rse_counter_history_table
+from rucio.daemons.common import run_daemon
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+graceful_stop = threading.Event()
+DAEMON_NAME = 'abacus-rse'
+
+
+def rse_update(once=False, sleep_time=10):
+    """
+    Main loop to check and update the RSE Counters.
+    """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=run_once,
+    )
+
+
+def run_once(heartbeat_handler, **_kwargs):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    # Select a bunch of rses for to update for this worker
+    start = time.time()  # NOQA
+    rse_ids = get_updated_rse_counters(total_workers=total_workers,
+                                       worker_number=worker_number)
+    logger(logging.DEBUG, 'Index query time %f size=%d' % (time.time() - start, len(rse_ids)))
+
+    # If the list is empty, sent the worker to sleep
+    if not rse_ids:
+        logger(logging.INFO, 'did not get any work')
+        return
+
+    for rse_id in rse_ids:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        if graceful_stop.is_set():
+            break
+        start_time = time.time()
+        update_rse_counter(rse_id=rse_id)
+        logger(logging.DEBUG, 'update of rse "%s" took %f' % (rse_id, time.time() - start_time))
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+
+    graceful_stop.set()
+
+
+def run(once=False, threads=1, fill_history_table=False, sleep_time=10):
+    """
+    Starts up the Abacus-RSE threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        logging.info('main: executing one iteration only')
+        rse_update(once)
+    else:
+        logging.info('main: starting threads')
+        threads = [threading.Thread(target=rse_update, kwargs={'once': once, 'sleep_time': sleep_time}) for i in
+                   range(0, threads)]
+        if fill_history_table:
+            threads.append(get_thread_with_periodic_running_function(3600, fill_rse_counter_history_table, graceful_stop))
+        [t.start() for t in threads]
+        logging.info('main: waiting for interrupts')
+        # Interruptible joins require a timeout.
+        while threads[0].is_alive():
+            [t.join(timeout=3.14) for t in threads]

rucio/daemons/atropos/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.