rucio 37.0.0rc1__py3-none-any.whl

rucio/core/message.py

@@ -0,0 +1,288 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import TYPE_CHECKING
+
+from sqlalchemy import delete, insert, or_, select, update
+from sqlalchemy.exc import IntegrityError
+
+from rucio.common.config import config_get_list
+from rucio.common.constants import MAX_MESSAGE_LENGTH, HermesService
+from rucio.common.exception import InvalidObject, RucioException
+from rucio.common.utils import APIEncoder, chunks
+from rucio.db.sqla import filter_thread_work
+from rucio.db.sqla.models import Message, MessageHistory
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from typing import Any, Optional
+
+    from sqlalchemy.orm import Session
+
+    MessageType = dict[str, Any]
+    MessagesListType = list[MessageType]
+
+
+@transactional_session
+def add_messages(messages: "MessagesListType", *, session: "Session") -> None:
+    """
+    Add a list of messages to be submitted asynchronously to a message broker.
+
+    For messages with payload bigger than MAX_MESSAGE_LENGTH, payload_nolimit is used instead of payload.
+    In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
+
+    :param messages: A list of dictionaries {'event_type': str, 'payload': dict}
+    :param session: The database session to use.
+    """
+    services = []
+    for service in config_get_list('hermes', 'services_list', raise_exception=False, default='activemq,email', session=session):
+        try:
+            HermesService(service.upper())
+        except ValueError as err:
+            raise RucioException(str(err))
+        services.append(service)
+
+    msgs = []
+    for message in messages:
+        event_type = message['event_type']
+        payload = message['payload']
+        for service in services:
+            msg = {'services': service, 'event_type': event_type}
+            try:
+                if event_type == 'email' and service != 'email':
+                    continue
+                if service == 'email' and event_type != 'email':
+                    continue
+                msg_payload = json.dumps(payload, cls=APIEncoder)
+                msg['payload'] = msg_payload
+                if len(msg_payload) > MAX_MESSAGE_LENGTH:
+                    msg['payload_nolimit'] = msg_payload
+                    msg['payload'] = 'nolimit'
+                msgs.append(msg)
+            except TypeError as err:  # noqa: F841
+                raise InvalidObject(f'Invalid JSON for payload: {err}')
+    for messages_chunk in chunks(msgs, 1000):
+        stmt = insert(
+            Message
+        )
+        session.execute(stmt, messages_chunk)
+
+
+@transactional_session
+def add_message(event_type: str, payload: dict, *, session: "Session") -> None:
+    """
+    Add a message to be submitted asynchronously to a message broker.
+
+    In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
+
+    :param event_type: The type of the event as a string, e.g., NEW_DID.
+    :param payload: The message payload. Will be persisted as JSON.
+    :param session: The database session to use.
+    """
+    add_messages([{'event_type': event_type, 'payload': payload}], session=session)
+
+
+@transactional_session
+def retrieve_messages(bulk: int = 1000,
+                      thread: "Optional[int]" = None,
+                      total_threads: "Optional[int]" = None,
+                      event_type: "Optional[str]" = None,
+                      lock: bool = False,
+                      old_mode: bool = True,
+                      service_filter: "Optional[str]" = None,
+                      *, session: "Session") -> "MessagesListType":
+    """
+    Retrieve up to $bulk messages.
+
+    :param bulk: Number of messages as an integer.
+    :param thread: Identifier of the caller thread as an integer.
+    :param total_threads: Maximum number of threads as an integer.
+    :param event_type: Return only specified event_type. If None, returns everything.
+    :param lock: Select exclusively some rows.
+    :param old_mode: If True, doesn't return email if event_type is None.
+    :param session: The database session to use.
+    :param service_filter: When a service is supplied this queries the database for messages for that service.
+
+    :returns messages: List of dictionaries {id, created_at, event_type, payload, services}
+    """
+    messages = []
+    try:
+        stmt_subquery = select(
+            Message.id
+        ).order_by(
+            Message.created_at
+        )
+        stmt_subquery = filter_thread_work(session=session, query=stmt_subquery, total_threads=total_threads, thread_id=thread)
+        if service_filter:
+            stmt_subquery = stmt_subquery.where(
+                Message.services == service_filter
+            )
+        if event_type:
+            stmt_subquery = stmt_subquery.where(
+                Message.event_type == event_type
+            )
+        elif old_mode:
+            stmt_subquery = stmt_subquery.where(
+                Message.event_type != 'email'
+            )
+
+        # Step 1:
+        # MySQL does not support limits in nested queries, limit on the outer query instead.
+        # This is not as performant, but the best we can get from MySQL.
+        # FIXME: SQLAlchemy generates wrong nowait MySQL8 statement for MySQL5
+        #        Remove once this is resolved in SQLAlchemy
+        stmt = select(
+            Message.id,
+            Message.created_at,
+            Message.event_type,
+            Message.payload,
+            Message.services
+        )
+        if session.bind.dialect.name == 'mysql':
+            stmt = stmt.where(
+                Message.id.in_(stmt_subquery)
+            )
+        else:
+            stmt_subquery = stmt_subquery.limit(
+                bulk
+            )
+            stmt = stmt.where(
+                Message.id.in_(stmt_subquery)
+            ).with_for_update(
+                nowait=True
+            )
+
+        # Step 2:
+        # MySQL does not support limits in nested queries, limit on the outer query instead.
+        # This is not as performant, but the best we can get from MySQL.
+        if session.bind.dialect.name == 'mysql':
+            stmt = stmt.limit(
+                bulk
+            )
+
+        # Step 3:
+        # Assemble message object
+        for id_, created_at, event_type, payload, services in session.execute(stmt).all():
+            message = {'id': id_,
+                       'created_at': created_at,
+                       'event_type': event_type,
+                       'services': services}
+
+            # Only switch SQL context when necessary
+            if payload == 'nolimit':
+                nolimit_stmt = select(
+                    Message.payload_nolimit
+                ).where(
+                    Message.id == id_
+                )
+                message['payload'] = json.loads(str(session.execute(nolimit_stmt).scalar_one()))
+            else:
+                message['payload'] = json.loads(str(payload))
+
+            messages.append(message)
+
+        return messages
+
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def delete_messages(messages: "MessagesListType", *, session: "Session") -> None:
+    """
+    Delete all messages with the given IDs, and archive them to the history.
+
+    :param messages: The messages to delete as a list of dictionaries.
+    """
+    message_condition = []
+    for message in messages:
+        message_condition.append(Message.id == message['id'])
+        if len(message['payload']) > MAX_MESSAGE_LENGTH:
+            message['payload_nolimit'] = message.pop('payload')
+
+    try:
+        if message_condition:
+            stmt = delete(
+                Message
+            ).prefix_with(
+                '/*+ INDEX(messages MESSAGES_ID_PK) */',
+                dialect='oracle'
+            ).where(
+                or_(*message_condition)
+            ).execution_options(
+                synchronize_session=False
+            )
+            session.execute(stmt)
+
+            stmt = insert(
+                MessageHistory
+            )
+            session.execute(stmt, messages)
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def truncate_messages(*, session: "Session") -> None:
+    """
+    Delete all stored messages. This is for internal purposes only.
+
+    :param session: The database session to use.
+    """
+
+    try:
+        stmt = delete(
+            Message
+        ).execution_options(
+            synchronize_session=False
+        )
+        session.execute(stmt)
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def update_messages_services(messages: "MessagesListType", services: str, *, session: "Session") -> None:
+    """
+    Update the services for all messages with the given IDs.
+
+    :param messages: The messages to delete as a list of dictionaries.
+    :param services: A coma separated string containing the list of services to report to.
+    :param session: The database session to use.
+    """
+    message_condition = []
+    for message in messages:
+        message_condition.append(Message.id == message['id'])
+        if len(message['payload']) > MAX_MESSAGE_LENGTH:
+            message['payload_nolimit'] = message.pop('payload')
+
+    try:
+        if message_condition:
+            stmt = update(
+                Message
+            ).prefix_with(
+                '/*+ INDEX(messages MESSAGES_ID_PK) */',
+                dialect='oracle'
+            ).where(
+                or_(*message_condition)
+            ).execution_options(
+                synchronize_session=False
+            ).values({
+                Message.services: services
+            })
+            session.execute(stmt)
+
+    except IntegrityError as err:
+        raise RucioException(err.args)

rucio/core/meta_conventions.py

@@ -0,0 +1,203 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from re import match
+from typing import TYPE_CHECKING, Optional, Union
+
+from sqlalchemy import and_, delete, select
+from sqlalchemy.exc import IntegrityError, NoResultFound
+
+from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
+from rucio.common.exception import Duplicate, InvalidObject, InvalidValueForKey, KeyNotFound, RucioException, UnsupportedKeyType, UnsupportedValueType
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import DIDType, KeyType
+from rucio.db.sqla.session import read_session, transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_key(key: str, key_type: Union[KeyType, str], value_type: Optional[str] = None, value_regexp: Optional[str] = None, *, session: "Session") -> None:
+    """
+    Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
+
+    :param key: the name for the new key.
+    :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
+    :param value_type: the type of the value, if defined.
+    :param value_regexp: the regular expression that values should match, if defined.
+    :param session: The database session in use.
+    """
+
+    # Check if value_type is supported
+    if value_type and value_type not in [str(t) for t in AUTHORIZED_VALUE_TYPES]:
+        raise UnsupportedValueType(f"The type '{value_type}' is not supported for values!")
+
+    # Convert key_type
+    if isinstance(key_type, str):
+        key_type = str(key_type)
+    else:
+        key_type = str(key_type.value)
+
+    if key_type == 'F':
+        key_type = 'FILE'
+    elif key_type == 'D':
+        key_type = 'DATASET'
+    elif key_type == 'C':
+        key_type = 'CONTAINER'
+
+    try:
+        key_type = KeyType(key_type)
+    except ValueError:
+        raise UnsupportedKeyType('The type \'%s\' is not supported for keys!' % str(key_type))
+
+    new_key = models.DIDMetaConventionsKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type)
+    try:
+        new_key.save(session=session)
+    except IntegrityError as error:
+        if ('UNIQUE constraint failed' in error.args[0]) \
+           or ('conflicts with persistent instance' in error.args[0]) \
+           or match('.*IntegrityError.*ORA-00001: unique constraint.*DID_KEYS_PK.*violated.*', error.args[0]) \
+           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*IntegrityError.*columns? key.*not unique.*', error.args[0]):
+            raise Duplicate(f"key '{key}' already exists!")
+        raise RucioException(error.args)
+
+
+@transactional_session
+def del_key(key: str, *, session: "Session") -> None:
+    """
+    Delete a key in the DID Metadata Conventions table.
+
+    :param key: the name for the key.
+    :param session: The database session in use.
+    """
+    stmt = delete(
+        models.DIDMetaConventionsKey
+    ).where(
+        models.DIDMetaConventionsKey.key == key
+    )
+    session.execute(stmt)
+
+
+@read_session
+def list_keys(*, session: "Session") -> list[str]:
+    """
+    Lists all keys for DID Metadata Conventions.
+
+    :param session: The database session in use.
+
+    :returns: A list containing all keys.
+    """
+    stmt = select(
+        models.DIDMetaConventionsKey.key
+    )
+    return list(session.execute(stmt).scalars().all())
+
+
+@transactional_session
+def add_value(key: str, value: str, *, session: "Session") -> None:
+    """
+    Adds a new value for a key in DID Metadata Convention.
+
+    :param key: the name for the key.
+    :param value: the value.
+    :param session: The database session in use.
+
+    :raises Duplicate: Key-Value pair exists
+    :raises KeyNotFound: Key not in metadata conventions table
+    :raises InvalidValueForKey: Value conflicts with rse expression for key values or does not have the correct type
+    """
+    new_value = models.DIDMetaConventionsConstraint(key=key, value=value)
+    try:
+        new_value.save(session=session)
+    except IntegrityError as error:
+        if ('UNIQUE constraint failed' in error.args[0]) \
+           or ('conflicts with persistent instance' in error.args[0]) \
+           or match('.*IntegrityError.*ORA-00001: unique constraint.*DID_KEYS_PK.*violated.*', error.args[0]) \
+           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*IntegrityError.*columns? key.*value.*not unique.*', error.args[0]):
+            raise Duplicate(f"key-value '{key}-{value}' already exists!")
+        if match('.*IntegrityError.*foreign key constraints? failed.*', error.args[0]):
+            raise KeyNotFound(f"key '{key}' does not exist!")
+        if match('.*IntegrityError.*ORA-02291: integrity constraint.*DID_MAP_KEYS_FK.*violated.*', error.args[0]):
+            raise KeyNotFound(f"key '{key}' does not exist!")
+        if error.args[0] == "(IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`rucio`.`did_key_map`, CONSTRAINT `DID_MAP_KEYS_FK` FOREIGN KEY (`key`) REFERENCES `did_keys` (`key`))')":
+            raise KeyNotFound(f"key '{key}' does not exist!")
+
+        raise RucioException(error.args)
+
+    statement = select(
+        models.DIDMetaConventionsKey,
+    ).where(
+        models.DIDMetaConventionsKey.key == key
+    )
+    query = session.execute(statement).scalar_one()
+
+    # Check value against regexp, if defined
+    if query.value_regexp and not match(query.value_regexp, value):
+        raise InvalidValueForKey(f"The value {value} for the key {key} does not match the regular expression {query.value_regexp}")
+
+    # Check value type, if defined
+    type_map = dict([(str(t), t) for t in AUTHORIZED_VALUE_TYPES])
+    if query.value_type and not isinstance(value, type_map.get(query.value_type)):  # type: ignore ; Typing error caused by 'isinstaince' not thinking types count as classes
+        raise InvalidValueForKey(f"The value {value} for the key {key} does not match the required type {query.value_type}")
+
+
+@read_session
+def list_values(key: str, *, session: "Session") -> list[str]:
+    """
+    Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
+
+    :param key: the name for the key.
+    :param session: The database session in use.
+
+    :returns: A list containing all values.
+    """
+    statement = select(
+        models.DIDMetaConventionsConstraint.value
+    ).where(
+        models.DIDMetaConventionsConstraint.key == key
+    )
+    return list(session.execute(statement).scalars().all())
+
+
+@read_session
+def validate_meta(meta: dict, did_type: DIDType, *, session: "Session") -> None:
+    """
+    Validates metadata for a did.
+
+    :param meta: the dictionary of metadata.
+    :param meta: the type of the did, e.g, DATASET, CONTAINER, FILE.
+    :param session: The database session in use.
+
+    :raises InvalidObject:
+    """
+    # For now only validate the datatype for datasets
+    key = 'datatype'
+    if did_type == DIDType.DATASET and key in meta:
+        try:
+            statement = select(
+                models.DIDMetaConventionsConstraint.value
+            ).where(
+                and_(models.DIDMetaConventionsConstraint.value == meta[key],
+                     models.DIDMetaConventionsConstraint.key == key)
+            )
+            session.execute(statement).one()
+        except NoResultFound:
+            raise InvalidObject(f"The value {meta[key]}' for the key {key} is not valid")