oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/datastreams/writers/attachments_service.py

@@ -1,118 +0,0 @@
-from io import BytesIO
-
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services.uow import UnitOfWork
-
-from ...uow import BulkUnitOfWork
-from ...utils.identity_utils import get_user_and_identity
-from ..types import StreamBatch, StreamEntry
-from ..utils import attachments_requests, get_file_service_for_record_service
-from . import BaseWriter
-from .utils import record_invenio_exceptions
-
-
-class AttachmentsServiceWriter(BaseWriter):
-    """Writes the entries to a repository instance using a Service object."""
-
-    def __init__(
-        self,
-        *,
-        service,
-        identity=None,
-        update=False,
-        **kwargs,
-    ):
-        """Constructor.
-        :param service_or_name: a service instance or a key of the
-                                service registry.
-        :param identity: access identity.
-        :param update: if True it will update records if they exist.
-        :param write_files: if True it will write files to the file service.
-        :param uow: UnitOfWork fully qualified class name or class to use for the unit of work.
-        """
-        super().__init__(**kwargs)
-
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        if isinstance(identity, str):
-            _, identity = get_user_and_identity(email=identity)
-        elif isinstance(identity, int):
-            _, identity = get_user_and_identity(user_id=identity)
-        self._identity = identity or system_identity
-        self._update = update
-
-        self._file_service = None
-        self._record_cls = getattr(service.config, "record_cls", None)
-
-        self._file_service = get_file_service_for_record_service(service)
-
-    def _get_stream_entry_id(self, entry: StreamEntry):
-        return entry.id
-
-    def write(self, batch: StreamBatch):
-        """Writes the input entry using the given service."""
-
-        with BulkUnitOfWork() as uow:
-            for entry in batch.entries:
-                if not entry.ok or entry.deleted or not entry.entry["files"]["enabled"]:
-                    continue
-                with record_invenio_exceptions(entry):
-                    self._write_attachments(entry, uow)
-
-            uow.commit()
-
-        return batch
-
-    def _write_attachments(self, stream_entry: StreamEntry, uow: UnitOfWork):
-        service_kwargs = {}
-        if uow:
-            service_kwargs["uow"] = uow
-        entry_id = self._get_stream_entry_id(stream_entry)
-
-        existing_files = self._file_service.list_files(self._identity, entry_id)
-        existing_files = {f["key"]: f for f in existing_files.entries}
-
-        for f in stream_entry.files:
-            if f.metadata["key"] in existing_files:
-                if not self._update:
-                    continue
-                # TODO: compare if the file should be deleted and re-created
-                # if so, delete the file and create again
-                self._file_service.delete_file(
-                    self._identity, entry_id, f.metadata["key"], **service_kwargs
-                )
-
-            self._file_service.init_files(
-                self._identity,
-                entry_id,
-                [{"key": f.metadata["key"]}],
-                **service_kwargs,
-            )
-            metadata = f.metadata.get("metadata", {})
-            if metadata:
-                self._file_service.update_file_metadata(
-                    self._identity,
-                    entry_id,
-                    file_key=f.metadata["key"],
-                    data=metadata,
-                    **service_kwargs,
-                )
-            self._file_service.set_file_content(
-                self._identity,
-                entry_id,
-                f.metadata["key"],
-                BytesIO(attachments_requests.get(f.content_url).content),
-                **service_kwargs,
-            )
-            self._file_service.commit_file(
-                self._identity, entry_id, f.metadata["key"], **service_kwargs
-            )
-        new_files_keys = set(f.metadata["key"] for f in stream_entry.files)
-
-        for existing_file_key in existing_files:
-            if existing_file_key not in new_files_keys:
-                self._file_service.delete_file(
-                    self._identity, entry_id, existing_file_key, **service_kwargs
-                )

oarepo_runtime/datastreams/writers/publish.py

@@ -1,70 +0,0 @@
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.datastreams.types import StreamBatch, StreamEntry
-from oarepo_runtime.datastreams.writers import BaseWriter
-from oarepo_runtime.datastreams.writers.utils import record_invenio_exceptions
-
-from ...utils.identity_utils import get_user_and_identity
-
-
-class PublishWriter(BaseWriter):
-    def __init__(
-        self,
-        *,
-        service,
-        request_name="publish_draft",
-        identity=None,
-        direct_call=True,
-        **kwargs
-    ):
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        self._service = service
-        if isinstance(identity, str):
-            _, identity = get_user_and_identity(email=identity)
-        elif isinstance(identity, int):
-            _, identity = get_user_and_identity(user_id=identity)
-        self._identity = identity or system_identity
-        self._request_name = request_name
-        self._direct_call = direct_call
-
-    def write(self, batch: StreamBatch) -> StreamBatch:
-        for entry in batch.ok_entries:
-            if entry.deleted:
-                continue
-
-            with record_invenio_exceptions(entry):
-                self._write_entry(entry)
-
-    def _write_entry(self, entry: StreamEntry):
-        if self._direct_call:
-            data = self._service.publish(self._identity, entry.id)
-        else:
-            data = self._publish_via_request(self._identity, entry.id)
-
-        entry.entry = data.to_dict()
-
-    def _publish_via_request(self, identity, entry_id):
-        from invenio_requests.proxies import (
-            current_requests_service as current_invenio_requests_service,
-        )
-        from oarepo_requests.proxies import current_oarepo_requests_service
-
-        draft = self._service.read_draft(identity, entry_id)
-        request = current_oarepo_requests_service.create(
-            identity=identity,
-            data=None,
-            request_type=self._request_name,
-            topic=draft._record,
-        )
-
-        submit_result = current_invenio_requests_service.execute_action(
-            identity, request.id, "submit"
-        )
-        accept_result = current_invenio_requests_service.execute_action(
-            identity, request.id, "accept"
-        )
-
-        return self._service.read(identity, draft["id"])

oarepo_runtime/datastreams/writers/service.py

@@ -1,175 +0,0 @@
-from invenio_access.permissions import system_identity
-from invenio_pidstore.errors import PIDDoesNotExistError
-from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services.uow import UnitOfWork
-from sqlalchemy.exc import NoResultFound
-
-from ...uow import BulkUnitOfWork
-from ...utils.identity_utils import get_user_and_identity
-from ..types import StreamBatch, StreamEntry, StreamEntryError
-from . import BaseWriter
-from .utils import record_invenio_exceptions
-
-
-class ServiceWriter(BaseWriter):
-    """Writes the entries to a repository instance using a Service object."""
-
-    def __init__(
-        self,
-        *,
-        service,
-        identity=None,
-        update=False,
-        **kwargs,
-    ):
-        """Constructor.
-        :param service_or_name: a service instance or a key of the
-                                service registry.
-        :param identity: access identity.
-        :param update: if True it will update records if they exist.
-        :param write_files: if True it will write files to the file service.
-        :param uow: UnitOfWork fully qualified class name or class to use for the unit of work.
-        """
-        super().__init__(**kwargs)
-
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        self._service = service
-        if isinstance(identity, str):
-            _, identity = get_user_and_identity(email=identity)
-        elif isinstance(identity, int):
-            _, identity = get_user_and_identity(user_id=identity)
-        self._identity = identity or system_identity
-        self._update = update
-
-    def _resolve(self, id_):
-        if hasattr(self._service, "read_draft"):
-            try:
-                # try to read the draft first
-                return self._service.read_draft(self._identity, id_)
-            except PIDDoesNotExistError:
-                pass
-            try:
-                # if the draft does not exist, read the published record
-                # and create a draft for it
-                rec = self._service.read(self._identity, id_)
-                return self._service.edit(self._identity, id_)
-            except PIDDoesNotExistError:
-                pass
-
-        else:
-            try:
-                return self._service.read(self._identity, id_)
-            except PIDDoesNotExistError:
-                pass
-            except NoResultFound:
-                # vocabularies do not raise a PIDDoesNotExistError, they raise sql exception
-                pass
-        return None
-
-    def _get_stream_entry_id(self, entry: StreamEntry):
-        return entry.id or entry.entry.get("id")
-
-    def write(self, batch: StreamBatch):
-        """Writes the input entry using the given service."""
-        with BulkUnitOfWork() as uow:
-            for entry in batch.entries:
-                if entry.filtered or entry.errors:
-                    continue
-                with record_invenio_exceptions(entry):
-                    if entry.deleted:
-                        self._delete_entry(entry, uow=uow)
-                    else:
-                        self._write_entry(entry, uow)
-            uow.commit()
-
-        return batch
-
-    def _write_entry(self, stream_entry: StreamEntry, uow: UnitOfWork):
-        entry = stream_entry.entry
-        service_kwargs = {}
-        if uow:
-            service_kwargs["uow"] = uow
-
-        do_create = True
-        repository_entry = None  # just to make linter happy
-
-        entry_id = self._get_stream_entry_id(stream_entry)
-
-        if entry_id:
-            if self._update:
-                repository_entry = self.try_update(entry_id, entry, **service_kwargs)
-                if repository_entry:
-                    do_create = False
-            else:
-                current = self._resolve(entry_id)
-                if current:
-                    do_create = False
-
-        if do_create:
-            repository_entry = self._service.create(
-                self._identity, entry, **service_kwargs
-            )
-
-        if repository_entry:
-            stream_entry.entry = repository_entry.data
-            stream_entry.id = repository_entry.id
-
-            stream_entry.context["revision_id"] = repository_entry._record.revision_id
-            if repository_entry.errors:
-                for err in repository_entry.errors:
-                    field = err.get("field")
-                    messages = err.get("messages")
-                    for message in messages:
-                        stream_entry.errors.append(
-                            StreamEntryError(
-                                code="validation", message=message, location=field
-                            )
-                        )
-
-    def try_update(self, entry_id, entry, **service_kwargs):
-        current = self._resolve(entry_id)
-        if current:
-            updated = dict(current.to_dict(), **entry)
-            # might raise exception here but that's ok - we know that the entry
-            # exists in db as it was _resolved
-            if hasattr(self._service, "update_draft"):
-                # try to update draft first
-                return self._service.update_draft(
-                    self._identity, entry_id, updated, **service_kwargs
-                )
-            else:
-                return self._service.update(
-                    self._identity, entry_id, updated, **service_kwargs
-                )
-
-    def _delete_entry(self, stream_entry: StreamEntry, uow=None):
-        entry_id = self._get_stream_entry_id(stream_entry)
-        if not entry_id:
-            return
-        service_kwargs = {}
-        if uow:
-            service_kwargs["uow"] = uow
-        deletion_exceptions = []
-        deletion_tries = 0
-
-        # if the service has drafts, try to delete it first
-        if hasattr(self._service, "delete_draft"):
-            # delete draft
-            deletion_tries += 1
-            try:
-                self._service.delete_draft(self._identity, entry_id, **service_kwargs)
-            except Exception as e:
-                deletion_exceptions.append(e)
-
-        # delete the record if it was published
-        deletion_tries += 1
-        try:
-            self._service.delete(self._identity, entry_id, **service_kwargs)
-        except Exception as e:
-            deletion_exceptions.append(e)
-
-        if len(deletion_exceptions) == deletion_tries:
-            # all deletion attempts failed
-            raise deletion_exceptions[-1]

oarepo_runtime/datastreams/writers/utils.py

@@ -1,30 +0,0 @@
-import contextlib
-
-from invenio_records.systemfields.relations.errors import InvalidRelationValue
-from marshmallow import ValidationError
-
-from oarepo_runtime.datastreams import StreamEntry
-
-from ..types import StreamEntry, StreamEntryError
-from .validation_errors import format_validation_error
-
-
-@contextlib.contextmanager
-def record_invenio_exceptions(stream_entry: StreamEntry):
-    try:
-        yield
-    except ValidationError as err:
-        validation_errors = format_validation_error(err.messages)
-        for err_path, err_value in validation_errors.items():
-            stream_entry.errors.append(
-                StreamEntryError(
-                    code="MARHSMALLOW", message=err_value, location=err_path
-                )
-            )
-    except InvalidRelationValue as err:
-        # TODO: better formatting for this kind of error
-        stream_entry.errors.append(
-            StreamEntryError.from_exception(err, message=err.args[0])
-        )
-    except Exception as err:
-        stream_entry.errors.append(StreamEntryError.from_exception(err))

oarepo_runtime/datastreams/writers/validation_errors.py

@@ -1,20 +0,0 @@
-def format_validation_error(err_messages):
-    msg = {}
-    for err, path in flatten_errors(err_messages, ""):
-        if path.endswith("._schema") and err == "Unknown field.":
-            continue
-        path = path.replace(".value.", ".")
-        msg[path] = err
-    return msg
-
-
-def flatten_errors(err_data, path):
-    if isinstance(err_data, (list, tuple)):
-        for err in err_data:
-            yield from flatten_errors(err, path)
-    elif isinstance(err_data, dict):
-        for k, v in err_data.items():
-            subpath = f"{path}.{k}" if path else k
-            yield from flatten_errors(v, subpath)
-    else:
-        yield str(err_data), path

oarepo_runtime/datastreams/writers/yaml.py

@@ -1,56 +0,0 @@
-from io import StringIO
-from pathlib import Path
-
-import yaml
-
-from oarepo_runtime.datastreams import BaseWriter, StreamBatch
-
-
-class YamlWriter(BaseWriter):
-    """Writes the entries to a YAML file."""
-
-    def __init__(self, *, target, base_path=None, **kwargs):
-        """Constructor.
-        :param file_or_path: path of the output file.
-        """
-        super().__init__(**kwargs)
-        if hasattr(target, "write"):
-            # opened file
-            self._file = target
-            self._stream = target
-        else:
-            if base_path:
-                self._file = Path(base_path).joinpath(target)
-            else:
-                self._file = target
-            self._stream = open(self._file, "w")
-
-        self._started = False
-
-    def write(self, batch: StreamBatch):
-        """Writes the input stream entry using a given service."""
-
-        for entry in batch.entries:
-            if not entry.ok:
-                continue
-
-            self._write_entry_separator()
-
-            try:
-                io = StringIO()
-                yaml.safe_dump(entry.entry, io)
-                self._stream.write(io.getvalue())
-            except Exception as e:
-                entry.errors.append(e)
-
-        return batch
-
-    def _write_entry_separator(self):
-        if not self._started:
-            self._started = True
-        else:
-            self._stream.write("---\n")
-
-    def finish(self):
-        """Finalizes writing"""
-        self._stream.close()

oarepo_runtime/ext_config.py

@@ -1,67 +0,0 @@
-from invenio_records_resources.services.custom_fields import BooleanCF
-
-from oarepo_runtime.datastreams.fixtures import default_config_generator
-from oarepo_runtime.datastreams.readers.excel import ExcelReader
-from oarepo_runtime.datastreams.readers.json import JSONLinesReader, JSONReader
-from oarepo_runtime.datastreams.readers.service import ServiceReader
-from oarepo_runtime.datastreams.readers.yaml import YamlReader
-from oarepo_runtime.datastreams.writers.attachments_file import AttachmentsFileWriter
-from oarepo_runtime.datastreams.writers.attachments_service import (
-    AttachmentsServiceWriter,
-)
-from oarepo_runtime.datastreams.writers.publish import PublishWriter
-from oarepo_runtime.datastreams.writers.service import ServiceWriter
-from oarepo_runtime.datastreams.writers.yaml import YamlWriter
-from oarepo_runtime.records.entity_resolvers import UserResolver
-from oarepo_runtime.services.config.permissions_presets import (
-    AuthenticatedPermissionPolicy,
-    EveryonePermissionPolicy,
-    OaiHarvesterPermissionPolicy,
-    ReadOnlyPermissionPolicy,
-)
-from oarepo_runtime.services.facets.facet_groups_names import facet_groups_names
-
-OAREPO_PERMISSIONS_PRESETS = {
-    "read_only": ReadOnlyPermissionPolicy,
-    "everyone": EveryonePermissionPolicy,
-    "oai_harvester": OaiHarvesterPermissionPolicy,
-    "authenticated": AuthenticatedPermissionPolicy,
-}
-
-
-DATASTREAMS_READERS = {
-    "excel": ExcelReader,
-    "yaml": YamlReader,
-    "json": JSONReader,
-    "json-lines": JSONLinesReader,
-    "service": ServiceReader,
-}
-
-DATASTREAMS_READERS_BY_EXTENSION = {
-    "xlsx": "excel",
-    "yaml": "yaml",
-    "yml": "yaml",
-    "json": "json",
-    "json5": "json",
-    "jsonl": "json-lines",
-}
-
-DATASTREAMS_WRITERS = {
-    "service": ServiceWriter,
-    "attachments_service": AttachmentsServiceWriter,
-    "yaml": YamlWriter,
-    "attachments_file": AttachmentsFileWriter,
-    "publish": PublishWriter,
-}
-
-DATASTREAMS_TRANSFORMERS = {}
-
-DATASTREAMS_CONFIG_GENERATOR = default_config_generator
-
-HAS_DRAFT_CUSTOM_FIELD = [BooleanCF("has_draft")]
-
-OAREPO_FACET_GROUP_NAME = facet_groups_names
-
-OWNER_ENTITY_RESOLVERS = [
-    UserResolver(),
-]

oarepo_runtime/i18n/__init__.py

@@ -1,3 +0,0 @@
-from flask_babel import get_locale, gettext, lazy_gettext
-
-__all__ = ("gettext", "lazy_gettext", "get_locale")

oarepo_runtime/info/check.py

@@ -1,95 +0,0 @@
-import json, random
-from flask import Response, Blueprint
-from flask import current_app as flask_current_app
-from invenio_db import db # sql alchemy session
-from invenio_cache import current_cache # redis
-import redis
-from invenio_search import current_search_client # opensearch
-from celery import current_app as celery_current_app, shared_task
-
-
-def check_db_connection():
-    if not has_database_connection():
-        return "DB connection failed"
-    else:
-        return "ok"
-
-def has_database_connection():
-    try:
-        db.session.execute("select * from alembic_version").scalar()
-        return True
-    except:
-        return False
-    finally:
-        db.session.rollback()
-
-def check_cache_connection():
-    try:
-        rand = random.randint(1, 1000)
-        if current_cache.set('mykey', str(rand)):
-            val = current_cache.get('mykey')
-            if val == str(rand):
-                return "ok"
-            else:
-                return "Cache returned unexpected value."
-        else:
-            return "Failed to set cache key."
-    except redis.exceptions.ConnectionError as e:
-        if isinstance(e.__cause__, ConnectionRefusedError):
-            return "Cache connection error"
-        return "Cache exception"
-    except Exception as other:
-        return str(other)
-
-
-
-#  https://github.com/celery/celery/issues/4283
-def check_message_queue():
-    from kombu.exceptions import OperationalError
-    try:
-        #celery_current_app.autodiscover_tasks(['oarepo_runtime.info'])
-        ping_task = celery_current_app.send_task('ping')
-        if ping_task.get(timeout=2) == 'pong':
-            return "ok"
-    except OperationalError as e:
-        if isinstance(e.__cause__, ConnectionRefusedError):
-            return "RabbitMQ connection refused"
-        return "Other RabbitMQ-related error"
-    except Exception as other:
-        return str(other)
-
-
-def check_opensearch():
-    try:
-        is_available = current_search_client.ping()
-        if is_available:
-            return "ok"
-        else:
-            return "OpenSearch is not available"
-    except Exception as e:
-        return str(e)
-
-
-blueprint = Blueprint("repository-check", __name__, url_prefix="/.well-known")
-
-@blueprint.route("/check")
-def check_connections() -> Response:
-    """
-    Function to check all necessary connections (database, redis, rabbitmq, opensearch)
-    Return Response 200 if everything is OK or Response 500 with json and all failed tests
-    """
-    check_list = {}
-    with flask_current_app.app_context():
-        check_list['db'] = check_db_connection()
-        check_list['cache'] = check_cache_connection()
-        check_list['mq'] = check_message_queue()
-        check_list['opensearch'] = check_opensearch()
-
-    failed_checks = {key: value for key, value in check_list.items() if value != 'ok'}
-
-    if failed_checks:
-        return Response(json.dumps(failed_checks), status=500, content_type='application/json')
-
-    return Response(json.dumps(check_list), status=200, content_type='application/json') # good
-
-