rucio 32.8.6__py3-none-any.whl

rucio/core/message.py

@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import TYPE_CHECKING
+
+from sqlalchemy import or_, delete, update, insert
+from sqlalchemy.exc import IntegrityError
+
+from rucio.common.config import config_get_list
+from rucio.common.constants import HermesService, MAX_MESSAGE_LENGTH
+from rucio.common.exception import InvalidObject, RucioException
+from rucio.common.utils import APIEncoder, chunks
+from rucio.db.sqla import filter_thread_work
+from rucio.db.sqla.models import Message, MessageHistory
+from rucio.db.sqla.session import transactional_session
+
+if TYPE_CHECKING:
+    from typing import Any, Optional
+    from sqlalchemy.orm import Session
+
+    MessageType = dict[str, Any]
+    MessagesListType = list[MessageType]
+
+
+@transactional_session
+def add_messages(messages: "MessagesListType", *, session: "Session") -> None:
+    """
+    Add a list of messages to be submitted asynchronously to a message broker.
+
+    For messages with payload bigger than MAX_MESSAGE_LENGTH, payload_nolimit is used instead of payload.
+    In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
+
+    :param messages: A list of dictionaries {'event_type': str, 'payload': dict}
+    :param session: The database session to use.
+    """
+    services = []
+    for service in config_get_list('hermes', 'services_list', raise_exception=False, default='activemq,email', session=session):
+        try:
+            HermesService(service.upper())
+        except ValueError as err:
+            raise RucioException(str(err))
+        services.append(service)
+
+    msgs = []
+    for message in messages:
+        event_type = message['event_type']
+        payload = message['payload']
+        for service in services:
+            msg = {'services': service, 'event_type': event_type}
+            try:
+                if event_type == 'email' and service != 'email':
+                    continue
+                if service == 'email' and event_type != 'email':
+                    continue
+                msg_payload = json.dumps(payload, cls=APIEncoder)
+                msg['payload'] = msg_payload
+                if len(msg_payload) > MAX_MESSAGE_LENGTH:
+                    msg['payload_nolimit'] = msg_payload
+                    msg['payload'] = 'nolimit'
+                msgs.append(msg)
+            except TypeError as err:  # noqa: F841
+                raise InvalidObject(f'Invalid JSON for payload: {err}')
+    for messages_chunk in chunks(msgs, 1000):
+        session.execute(insert(Message), messages_chunk)
+
+
+@transactional_session
+def add_message(event_type: str, payload: dict, *, session: "Session") -> None:
+    """
+    Add a message to be submitted asynchronously to a message broker.
+
+    In the case of nolimit, a placeholder string is written to the NOT NULL payload column.
+
+    :param event_type: The type of the event as a string, e.g., NEW_DID.
+    :param payload: The message payload. Will be persisted as JSON.
+    :param session: The database session to use.
+    """
+    add_messages([{'event_type': event_type, 'payload': payload}], session=session)
+
+
+@transactional_session
+def retrieve_messages(bulk: int = 1000,
+                      thread: "Optional[int]" = None,
+                      total_threads: "Optional[int]" = None,
+                      event_type: "Optional[str]" = None,
+                      lock: bool = False,
+                      old_mode: bool = True,
+                      *, session: "Session") -> "MessagesListType":
+    """
+    Retrieve up to $bulk messages.
+
+    :param bulk: Number of messages as an integer.
+    :param thread: Identifier of the caller thread as an integer.
+    :param total_threads: Maximum number of threads as an integer.
+    :param event_type: Return only specified event_type. If None, returns everything.
+    :param lock: Select exclusively some rows.
+    :param old_mode: If True, doesn't return email if event_type is None.
+    :param session: The database session to use.
+
+    :returns messages: List of dictionaries {id, created_at, event_type, payload, services}
+    """
+    messages = []
+    try:
+        subquery = session.query(Message.id)
+        subquery = filter_thread_work(session=session, query=subquery, total_threads=total_threads, thread_id=thread)
+        if event_type:
+            subquery = subquery.filter_by(event_type=event_type)
+        elif old_mode:
+            subquery = subquery.filter(Message.event_type != 'email')
+
+        # Step 1:
+        # MySQL does not support limits in nested queries, limit on the outer query instead.
+        # This is not as performant, but the best we can get from MySQL.
+        # FIXME: SQLAlchemy generates wrong nowait MySQL8 statement for MySQL5
+        #        Remove once this is resolved in SQLAlchemy
+        if session.bind.dialect.name == 'mysql':
+            subquery = subquery.order_by(Message.created_at)
+            query = session.query(Message.id,
+                                  Message.created_at,
+                                  Message.event_type,
+                                  Message.payload,
+                                  Message.services)\
+                           .filter(Message.id.in_(subquery))
+        else:
+            subquery = subquery.order_by(Message.created_at).limit(bulk)
+            query = session.query(Message.id,
+                                  Message.created_at,
+                                  Message.event_type,
+                                  Message.payload,
+                                  Message.services)\
+                           .filter(Message.id.in_(subquery))\
+                           .with_for_update(nowait=True)
+
+        # Step 2:
+        # MySQL does not support limits in nested queries, limit on the outer query instead.
+        # This is not as performant, but the best we can get from MySQL.
+        if session.bind.dialect.name == 'mysql':
+            query = query.limit(bulk)
+
+        # Step 3:
+        # Assemble message object
+        for id_, created_at, event_type, payload, services in query:
+            message = {'id': id_,
+                       'created_at': created_at,
+                       'event_type': event_type,
+                       'services': services}
+
+            # Only switch SQL context when necessary
+            if payload == 'nolimit':
+                nolimit_query = session.query(Message.payload_nolimit).filter(Message.id == id_).one()[0]
+                message['payload'] = json.loads(str(nolimit_query))
+            else:
+                message['payload'] = json.loads(str(payload))
+
+            messages.append(message)
+
+        return messages
+
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def delete_messages(messages: "MessagesListType", *, session: "Session") -> None:
+    """
+    Delete all messages with the given IDs, and archive them to the history.
+
+    :param messages: The messages to delete as a list of dictionaries.
+    """
+    message_condition = []
+    for message in messages:
+        message_condition.append(Message.id == message['id'])
+        if len(message['payload']) > MAX_MESSAGE_LENGTH:
+            message['payload_nolimit'] = message.pop('payload')
+
+    try:
+        if message_condition:
+            stmt = delete(Message).\
+                prefix_with("/*+ index(messages MESSAGES_ID_PK) */", dialect='oracle').\
+                where(or_(*message_condition)).\
+                execution_options(synchronize_session=False)
+            session.execute(stmt)
+
+            session.execute(insert(MessageHistory), messages)
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def truncate_messages(*, session: "Session") -> None:
+    """
+    Delete all stored messages. This is for internal purposes only.
+
+    :param session: The database session to use.
+    """
+
+    try:
+        session.query(Message).delete(synchronize_session=False)
+    except IntegrityError as e:
+        raise RucioException(e.args)
+
+
+@transactional_session
+def update_messages_services(messages: "MessagesListType", services: str, *, session: "Session") -> None:
+    """
+    Update the services for all messages with the given IDs.
+
+    :param messages: The messages to delete as a list of dictionaries.
+    :param services: A coma separated string containing the list of services to report to.
+    :param session: The database session to use.
+    """
+    message_condition = []
+    for message in messages:
+        message_condition.append(Message.id == message['id'])
+        if len(message['payload']) > MAX_MESSAGE_LENGTH:
+            message['payload_nolimit'] = message.pop('payload')
+
+    try:
+        if message_condition:
+            stmt = update(Message).\
+                prefix_with("/*+ index(messages MESSAGES_ID_PK) */", dialect='oracle').\
+                where(or_(*message_condition)).\
+                execution_options(synchronize_session=False).\
+                values(services=services)
+            session.execute(stmt)
+
+    except IntegrityError as err:
+        raise RucioException(err.args)

rucio/core/meta.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from re import match
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm.exc import NoResultFound
+
+from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
+from rucio.common.exception import (Duplicate, RucioException,
+                                    KeyNotFound, InvalidValueForKey, UnsupportedValueType,
+                                    InvalidObject, UnsupportedKeyType)
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import DIDType, KeyType
+from rucio.db.sqla.session import read_session, transactional_session
+
+if TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
+
+@transactional_session
+def add_key(key, key_type, value_type=None, value_regexp=None, *, session: "Session"):
+    """
+    Adds a new allowed key.
+
+    :param key: the name for the new key.
+    :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
+    :param value_type: the type of the value, if defined.
+    :param value_regexp: the regular expression that values should match, if defined.
+    :param session: The database session in use.
+    """
+
+    # Check if value_type is supported
+    if value_type and value_type not in [str(t) for t in AUTHORIZED_VALUE_TYPES]:
+        raise UnsupportedValueType(f"The type '{value_type}' is not supported for values!")
+
+    # Convert key_type
+    if isinstance(key_type, str):
+        key_type = str(key_type)
+    else:
+        key_type = str(key_type.value)
+
+    if key_type == 'F':
+        key_type = 'FILE'
+    elif key_type == 'D':
+        key_type = 'DATASET'
+    elif key_type == 'C':
+        key_type = 'CONTAINER'
+
+    try:
+        key_type = KeyType(key_type)
+    except ValueError:
+        raise UnsupportedKeyType('The type \'%s\' is not supported for keys!' % str(key_type))
+
+    new_key = models.DIDKey(key=key, value_type=value_type and str(value_type), value_regexp=value_regexp, key_type=key_type)
+    try:
+        new_key.save(session=session)
+    except IntegrityError as error:
+        if ('UNIQUE constraint failed' in error.args[0]) \
+           or ('conflicts with persistent instance' in error.args[0]) \
+           or match('.*IntegrityError.*ORA-00001: unique constraint.*DID_KEYS_PK.*violated.*', error.args[0]) \
+           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*IntegrityError.*columns? key.*not unique.*', error.args[0]):
+            raise Duplicate(f"key '{key}' already exists!")
+        raise
+
+
+@transactional_session
+def del_key(key, *, session: "Session"):
+    """
+    Deletes a key.
+
+    :param key: the name for the key.
+    :param session: The database session in use.
+    """
+    session.query(models.DIDKey).filter(key == key).delete()
+
+
+@read_session
+def list_keys(*, session: "Session"):
+    """
+    Lists all keys.
+
+    :param session: The database session in use.
+
+    :returns: A list containing all keys.
+    """
+    key_list = []
+    query = session.query(models.DIDKey)
+    for row in query:
+        key_list.append(row.key)
+    return key_list
+
+
+@transactional_session
+def add_value(key, value, *, session: "Session"):
+    """
+    Adds a new value to a key.
+
+    :param key: the name for the key.
+    :param value: the value.
+    :param session: The database session in use.
+    """
+    new_value = models.DIDKeyValueAssociation(key=key, value=value)
+    try:
+        new_value.save(session=session)
+    except IntegrityError as error:
+        if ('UNIQUE constraint failed' in error.args[0]) \
+           or ('conflicts with persistent instance' in error.args[0]) \
+           or match('.*IntegrityError.*ORA-00001: unique constraint.*DID_KEYS_PK.*violated.*', error.args[0]) \
+           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*IntegrityError.*columns? key.*value.*not unique.*', error.args[0]):
+            raise Duplicate(f"key-value '{key}-{value}' already exists!")
+        if match('.*IntegrityError.*foreign key constraints? failed.*', error.args[0]):
+            raise KeyNotFound(f"key '{key}' does not exist!")
+        if match('.*IntegrityError.*ORA-02291: integrity constraint.*DID_MAP_KEYS_FK.*violated.*', error.args[0]):
+            raise KeyNotFound(f"key '{key}' does not exist!")
+        if error.args[0] == "(IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`rucio`.`did_key_map`, CONSTRAINT `DID_MAP_KEYS_FK` FOREIGN KEY (`key`) REFERENCES `did_keys` (`key`))')":
+            raise KeyNotFound(f"key '{key}' does not exist!")
+
+        raise RucioException(error.args)
+
+    k = session.query(models.DIDKey).filter_by(key=key).one()
+
+    # Check value against regexp, if defined
+    if k.value_regexp and not match(k.value_regexp, value):
+        raise InvalidValueForKey("The value '%s' for the key '%s' does not match the regular expression '%s'" % (value, key, k.value_regexp))
+
+    # Check value type, if defined
+    type_map = dict([(str(t), t) for t in AUTHORIZED_VALUE_TYPES])
+    if k.value_type and not isinstance(value, type_map.get(k.value_type)):
+        raise InvalidValueForKey("The value '%s' for the key '%s' does not match the required type '%s'" % (value, key, k.value_type))
+
+
+@read_session
+def list_values(key, *, session: "Session"):
+    """
+    Lists all values for a key.
+
+    :param key: the name for the key.
+    :param session: The database session in use.
+
+    :returns: A list containing all values.
+    """
+    value_list = []
+    query = session.query(models.DIDKeyValueAssociation).filter_by(key=key)
+    for row in query:
+        value_list.append(row.value)
+    return value_list
+
+
+@read_session
+def validate_meta(meta, did_type, *, session: "Session"):
+    """
+    Validates metadata for a did.
+
+    :param meta: the dictionary of metadata.
+    :param meta: the type of the did, e.g, DATASET, CONTAINER, FILE.
+    :param session: The database session in use.
+
+    :returns: True
+    """
+    # For now only validate the datatype for datasets
+    key = 'datatype'
+    if did_type == DIDType.DATASET and key in meta:
+        try:
+            session.query(models.DIDKeyValueAssociation.value).\
+                filter_by(key=key).\
+                filter_by(value=meta[key]).\
+                one()
+        except NoResultFound:
+            print("The value '%s' for the key '%s' is not valid" % (meta[key], key))
+            raise InvalidObject("The value '%s' for the key '%s' is not valid" % (meta[key], key))