oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime/datastreams/types.py

@@ -1,323 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import dataclasses
-import json
-import logging
-import textwrap
-import traceback
-from typing import Any, Dict, List, Optional, Union
-
-from .errors import DataStreamError
-from .json import JSONObject
-
-log = logging.getLogger("datastreams")
-
-
-@dataclasses.dataclass
-class StreamEntryError:
-    code: str
-    message: str
-    location: Optional[str] = None
-    info: Union[JSONObject, None] = None
-
-    @classmethod
-    def from_exception(
-        cls, exc: Exception, limit=30, message=None, location=None, info=None, code=None
-    ):
-        if isinstance(exc, DataStreamError):
-            return cls(
-                code=exc.code,
-                message=exc.message,
-                location=exc.location,
-                info=exc.detail,
-            )
-
-        # can not use format_exception here as the signature is different for python 3.9 and python 3.10
-        stack = traceback.format_exc(limit=limit)
-        if message:
-            formatted_exception = message
-        elif hasattr(exc, "format_exception"):
-            formatted_exception = exc.format_exception()
-        else:
-            formatted_exception = str(exc)
-
-        return cls(
-            code=code or getattr(exc, "type", type(exc).__name__),
-            message=formatted_exception,
-            location=location,
-            info={
-                "message": str(exc),
-                "exception": type(exc).__name__,
-                "stack": stack,
-                **(info or {}),
-            },
-        )
-
-    @property
-    def json(self) -> JSONObject:
-        ret = {}
-        if self.code:
-            ret["code"] = self.code
-        if self.message:
-            ret["message"] = self.message
-        if self.location:
-            ret["location"] = self.location
-        if self.info:
-            ret["info"] = self.info
-        return ret
-
-    @classmethod
-    def from_json(cls, js: JSONObject):
-        if js is None:
-            return None
-        return cls(
-            code=js.get("code"),
-            message=js.get("message"),
-            location=js.get("location"),
-            info=js.get("info"),
-        )
-
-    def __str__(self):
-        formatted_info = textwrap.indent(
-            json.dumps(self.info or {}, ensure_ascii=False, indent=4), prefix="    "
-        )
-        return f"{self.code}:{self.location if self.location else ''} {self.message}\n{formatted_info}"
-
-    def __repr__(self):
-        return str(self)
-
-
-@dataclasses.dataclass
-class StreamEntryFile:
-    metadata: JSONObject
-    content_url: str
-    "data url with the content of the file or any other resolvable url"
-
-    @property
-    def json(self) -> JSONObject:
-        return {
-            "metadata": self.metadata,
-            "content_url": self.content_url,
-        }
-
-    @classmethod
-    def from_json(cls, js: JSONObject):
-        return cls(
-            metadata=js["metadata"],
-            content_url=js["content_url"],
-        )
-
-
-@dataclasses.dataclass
-class StreamEntry:
-    """Object to encapsulate streams processing."""
-
-    entry: JSONObject
-    files: List[StreamEntryFile] = dataclasses.field(default_factory=list)
-    seq: int = 0
-    id: Optional[str] = None
-    filtered: bool = False
-    deleted: bool = False
-    errors: List[StreamEntryError] = dataclasses.field(default_factory=list)
-    context: JSONObject = dataclasses.field(default_factory=dict)
-
-    @property
-    def ok(self):
-        return not self.filtered and not self.errors
-
-    @property
-    def json(self) -> JSONObject:
-        return {
-            "id": self.id,
-            "entry": self.entry,
-            "filtered": self.filtered,
-            "deleted": self.deleted,
-            "errors": [x.json for x in self.errors],
-            "context": self.context,
-            "seq": self.seq,
-            "files": [x.json for x in self.files],
-        }
-
-    @classmethod
-    def from_json(cls, js):
-        return cls(
-            id=js["id"],
-            entry=js["entry"],
-            filtered=js["filtered"],
-            deleted=js["deleted"],
-            errors=[StreamEntryError.from_json(x) for x in js["errors"]],
-            context=js["context"],
-            seq=js["seq"],
-            files=[StreamEntryFile.from_json(x) for x in js["files"]],
-        )
-
-    def __str__(self):
-        ret = [
-            f"Entry #{self.seq}: id {self.id or 'not yet set'}, filtered: {self.filtered}, deleted: {self.deleted}",
-            "Content:",
-            textwrap.indent(
-                json.dumps(self.entry, ensure_ascii=False, indent=4), "    "
-            ),
-            "Context:",
-            textwrap.indent(
-                json.dumps(self.context, ensure_ascii=False, indent=4), "    "
-            ),
-        ]
-        if self.errors:
-            ret.append("Errors:")
-            for error in self.errors:
-                ret.append(textwrap.indent(str(error), "    "))
-        return "\n".join(ret)
-
-
-@dataclasses.dataclass
-class StreamBatch:
-    entries: List[StreamEntry]
-    context: Dict[str, Any] = dataclasses.field(default_factory=dict)
-    last: bool = False
-    seq: int = 0
-    errors: List[StreamEntryError] = dataclasses.field(default_factory=list)
-
-    @property
-    def ok_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.ok]
-
-    @property
-    def failed_entries(self):
-        if self.errors:
-            return self.entries
-        return [x for x in self.entries if x.errors]
-
-    @property
-    def skipped_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.filtered]
-
-    @property
-    def deleted_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.deleted]
-
-    @property
-    def json(self):
-        return {
-            "entries": [x.json for x in self.entries],
-            "context": self.context,
-            "last": self.last,
-            "seq": self.seq,
-            "errors": [x.json for x in self.errors],
-        }
-
-    @classmethod
-    def from_json(cls, js):
-        if js is None:
-            return None
-        try:
-            [StreamEntry.from_json(x) for x in js["entries"]]
-        except:
-            log.exception("Exception parsing %s", js)
-            raise
-        return cls(
-            entries=[StreamEntry.from_json(x) for x in js["entries"]],
-            context=js["context"],
-            last=js["last"],
-            seq=js["seq"],
-            errors=[StreamEntryError.from_json(x) for x in js["errors"]],
-        )
-
-
-class DataStreamCallback:
-    def __init__(self, log_error_entry=False):
-        self.log_error_entry = log_error_entry
-
-    def batch_started(self, batch):
-        log.info("Batch started: %s", batch.seq)
-        if log.isEnabledFor(logging.DEBUG):
-            log.debug("Content: %s", batch)
-
-    def batch_finished(self, batch: StreamBatch):
-        log.info("Batch finished: %s", batch.seq)
-        if log.isEnabledFor(logging.DEBUG):
-            log.debug("Content: %s", batch)
-        for err in batch.errors:
-            log.error("Failed batch: %s: %s", err, batch.seq)
-        if self.log_error_entry:
-            for entry in batch.entries:
-                if entry.errors:
-                    log.error("Failed entry: %s in batch %s", entry, batch.seq)
-
-    def reader_error(self, reader, exception):
-        log.error("Reader error: %s: %s", reader, exception)
-
-    def transformer_error(self, batch, transformer, exception):
-        log.error("Transformer error: %s: %s", transformer, exception)
-
-    def writer_error(self, batch, writer, exception):
-        log.error("Writer error: %s: %s", writer, exception)
-
-
-class StatsKeepingDataStreamCallback(DataStreamCallback):
-    def __init__(self, log_error_entry=False):
-        super().__init__(log_error_entry=log_error_entry)
-
-        self.started_batches_count = 0
-        self.finished_batches_count = 0
-        self.ok_entries_count = 0
-        self.filtered_entries_count = 0
-        self.deleted_entries_count = 0
-        self.failed_entries_count = 0
-        self.reader_errors_count = 0
-        self.transformer_errors_count = 0
-        self.writer_errors_count = 0
-
-    def batch_started(self, batch):
-        super().batch_started(batch)
-        self.started_batches_count += 1
-
-    def batch_finished(self, batch: StreamBatch):
-        super().batch_finished(batch)
-        self.finished_batches_count += 1
-        for entry in batch.entries:
-            if entry.ok:
-                self.ok_entries_count += 1
-            if entry.filtered:
-                self.filtered_entries_count += 1
-            if entry.deleted:
-                self.deleted_entries_count += 1
-            if entry.errors:
-                self.failed_entries_count += 1
-
-    def reader_error(self, reader, exception):
-        super().reader_error(reader, exception)
-        self.reader_errors_count += 1
-
-    def transformer_error(self, batch, transformer, exception):
-        super().transformer_error(batch, transformer, exception)
-        self.transformer_errors_count += 1
-
-    def writer_error(self, batch, writer, exception):
-        super().writer_error(batch, writer, exception)
-        self.writer_errors_count += 1
-
-    def stats(self):
-        ret = [f"{self.finished_batches_count} batches finished"]
-        if self.ok_entries_count:
-            ret.append(f"ok: {self.ok_entries_count}")
-        if self.deleted_entries_count:
-            ret.append(f"deleted: {self.deleted_entries_count}")
-        if self.filtered_entries_count:
-            ret.append(f"filtered: {self.filtered_entries_count}")
-        if self.failed_entries_count:
-            ret.append(f"failed: {self.failed_entries_count}")
-        if self.reader_errors_count:
-            ret.append(f"reader errors: {self.reader_errors_count}")
-        if self.transformer_errors_count:
-            ret.append(f"transformer errors: {self.transformer_errors_count}")
-        if self.writer_errors_count:
-            ret.append(f"writer errors: {self.writer_errors_count}")
-        return ", ".join(ret)

oarepo_runtime/datastreams/utils.py

@@ -1,131 +0,0 @@
-from base64 import b64decode
-
-import requests
-from deprecated import deprecated
-from flask import current_app
-from invenio_drafts_resources.services import RecordService as DraftRecordService
-from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services import FileService, RecordService
-from invenio_records_resources.services.records.results import RecordItem
-from requests import PreparedRequest, Response
-from requests.adapters import BaseAdapter
-
-
-def get_record_service_for_record(record):
-    if not record:
-        return None
-    if "OAREPO_PRIMARY_RECORD_SERVICE" in current_app.config:
-        return get_record_service_for_record_class(type(record))
-    else:
-        return get_record_service_for_record_deprecated(record)
-
-
-def get_record_service_for_record_class(record_cls):
-    service_id = current_app.config["OAREPO_PRIMARY_RECORD_SERVICE"][record_cls]
-    return current_service_registry.get(service_id)
-
-
-@deprecated(
-    version="1.5.43", reason="Please recompile model to remove this deprecation warning"
-)
-def get_record_service_for_record_deprecated(record):
-    if getattr(record, "is_draft", False):
-        record_name = "draft_cls"
-        expect_draft_service = True
-    else:
-        record_name = "record_cls"
-        expect_draft_service = False
-
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, RecordService):
-            continue
-        if isinstance(svc, FileService):
-            continue
-        is_draft_service = isinstance(svc, DraftRecordService)
-        if is_draft_service != expect_draft_service:
-            continue
-        service_record = getattr(svc, record_name, None)
-        if service_record == type(record):
-            return svc
-
-def get_file_service_for_record_class(record_class):
-    if not record_class:
-        return None
-
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, FileService):
-            continue
-        if svc.record_cls != record_class:
-            continue
-        return svc
-
-def get_file_service_for_file_record_class(file_record_class):
-    record_class = file_record_class.record_cls
-    return get_file_service_for_record_class(record_class)
-
-def get_file_service_for_record_service(
-    record_service, check_draft_files=True, record=None
-):
-    if isinstance(record, RecordItem):
-        record = record._record
-    if record and getattr(record, "is_draft", False) is False:
-        check_draft_files = False
-    if (
-        check_draft_files
-        and hasattr(record_service, "draft_files")
-        and isinstance(record_service.draft_files, FileService)
-    ):
-        return record_service.draft_files
-    if hasattr(record_service, "files") and isinstance(
-        record_service.files, FileService
-    ):
-        return record_service.files
-    return get_file_service_for_record_class(
-        getattr(record_service.config, "record_cls", None)
-    )
-
-
-def get_record_service_for_file_service(file_service, record=None):
-    if record and getattr(record, "is_draft", False):
-        record_name = "draft_cls"
-        expect_draft_service = True
-    else:
-        record_name = "record_cls"
-        expect_draft_service = False
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, RecordService):
-            continue
-        is_draft_service = isinstance(svc, DraftRecordService)
-        if is_draft_service != expect_draft_service:
-            continue
-        service_record = getattr(svc, record_name, None)
-        if service_record == file_service.record_cls:
-            return svc
-
-    raise KeyError(
-        f"Could not get service for file service {file_service}, draft {expect_draft_service}"
-    )
-
-
-class DataAdapter(BaseAdapter):
-    def send(
-        self,
-        request: PreparedRequest,
-        stream=False,
-        timeout=None,
-        verify=True,
-        cert=None,
-        proxies=None,
-    ):
-        data = request.url.replace("data:", "")
-        resp = Response()
-        resp.status_code = 200
-        resp._content = b64decode(data)
-        return resp
-
-    def close(self):
-        pass
-
-
-attachments_requests = requests.Session()
-attachments_requests.mount("data:", DataAdapter())

oarepo_runtime/datastreams/writers/__init__.py

@@ -1,21 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Union
-
-from oarepo_runtime.datastreams.types import StreamBatch
-
-
-class BaseWriter(ABC):
-    """Base writer."""
-
-    def __init__(self, **kwargs) -> None:
-        """kwargs for extensions"""
-
-    @abstractmethod
-    def write(self, batch: StreamBatch) -> Union[StreamBatch, None]:
-        """Writes the input entry to the target output.
-        :returns: nothing
-                  Raises WriterException in case of errors.
-        """
-
-    def finish(self):
-        pass

oarepo_runtime/datastreams/writers/attachments_file.py

@@ -1,92 +0,0 @@
-import os
-from pathlib import Path
-
-import yaml
-
-from oarepo_runtime.datastreams import StreamBatch, StreamEntry
-
-from ..utils import attachments_requests
-from . import BaseWriter
-
-
-class AttachmentsFileWriter(BaseWriter):
-    """
-    Writes the files and its metadata into subdirectories.
-
-    The path will be files/<record-id>/<file-id>/metadata.yaml for technical metadata
-    and files/<record-id>/<file-id>/<key> for the data.
-
-    If the data key is "metadata.yaml", then "metadata" will be placed to "metametadata.yaml"
-    """
-
-    def __init__(self, *, target, base_path=None, **kwargs):
-        """Constructor.
-        :param file_or_path: path of the output file.
-        """
-        super().__init__(**kwargs)
-        self._grouping = 3
-        self._min_padding = 3
-        if base_path:
-            self._dir = Path(base_path).joinpath(target)
-        else:
-            self._dir = Path(target)
-
-    def write(self, batch: StreamBatch, *args, **kwargs):
-        """Writes the input stream entry using a given service."""
-        """
-        context looks like: {
-            'files': [
-            {'metadata': {'updated': '...', 'mimetype': 'image/png', 'storage_class': 'L', 'file_id': '', 
-                          'links': {...}, 'size': 27, 'status': 'completed', 'version_id': '...', 
-                          'bucket_id': '...', 'metadata': None, 'key': 'test.png', 
-                          'checksum': 'md5:...', 'created': '...'}, 
-             'content': b'test file content: test.png'}]}
-        """
-        for entry in batch.entries:
-            if entry.ok and entry.files:
-                self.write_entry(entry)
-
-    def write_entry(self, entry: StreamEntry):
-        dirname = self._dir.joinpath(format_serial(entry.seq)) / "data"
-        dirname.mkdir(parents=True, exist_ok=False)
-        file_keys = []
-        files_metadata = []
-        for fn_idx, fn in enumerate(entry.files):
-            md = {**fn.metadata}
-            content = attachments_requests.get(fn.content_url).content
-            # cleanup
-            md.pop("storage_class", None)
-            md.pop("file_id", None)
-            md.pop("links", None)
-            md.pop("status", None)
-            md.pop("version_id", None)
-            md.pop("bucket_id", None)
-            key = md["key"]
-            file_keys.append(key)
-            files_metadata.append(md)
-            (dirname / key).write_bytes(content)
-        metadata_key = "metadata.yaml"
-        while metadata_key in file_keys:
-            metadata_key = "meta_" + metadata_key
-        with open(dirname / metadata_key, "w") as f:
-            yaml.safe_dump_all(files_metadata, f)
-        return entry
-
-    def finish(self):
-        """Finalizes writing"""
-
-
-def format_serial(serial_no):
-    grouping = 3
-    min_padding = 3
-    serial_no = str(serial_no)
-    formatted_length = max(min_padding, len(serial_no))
-    while formatted_length % grouping:
-        formatted_length += 1
-    padded_serial = serial_no.zfill(formatted_length)
-    return os.sep.join(
-        [
-            padded_serial[i : i + grouping]
-            for i in range(0, len(padded_serial), grouping)
-        ]
-    )

oarepo_runtime/datastreams/writers/attachments_service.py

@@ -1,118 +0,0 @@
-from io import BytesIO
-
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services.uow import UnitOfWork
-
-from ...uow import BulkUnitOfWork
-from ...utils.identity_utils import get_user_and_identity
-from ..types import StreamBatch, StreamEntry
-from ..utils import attachments_requests, get_file_service_for_record_service
-from . import BaseWriter
-from .utils import record_invenio_exceptions
-
-
-class AttachmentsServiceWriter(BaseWriter):
-    """Writes the entries to a repository instance using a Service object."""
-
-    def __init__(
-        self,
-        *,
-        service,
-        identity=None,
-        update=False,
-        **kwargs,
-    ):
-        """Constructor.
-        :param service_or_name: a service instance or a key of the
-                                service registry.
-        :param identity: access identity.
-        :param update: if True it will update records if they exist.
-        :param write_files: if True it will write files to the file service.
-        :param uow: UnitOfWork fully qualified class name or class to use for the unit of work.
-        """
-        super().__init__(**kwargs)
-
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        if isinstance(identity, str):
-            _, identity = get_user_and_identity(email=identity)
-        elif isinstance(identity, int):
-            _, identity = get_user_and_identity(user_id=identity)
-        self._identity = identity or system_identity
-        self._update = update
-
-        self._file_service = None
-        self._record_cls = getattr(service.config, "record_cls", None)
-
-        self._file_service = get_file_service_for_record_service(service)
-
-    def _get_stream_entry_id(self, entry: StreamEntry):
-        return entry.id
-
-    def write(self, batch: StreamBatch):
-        """Writes the input entry using the given service."""
-
-        with BulkUnitOfWork() as uow:
-            for entry in batch.entries:
-                if not entry.ok or entry.deleted or not entry.entry["files"]["enabled"]:
-                    continue
-                with record_invenio_exceptions(entry):
-                    self._write_attachments(entry, uow)
-
-            uow.commit()
-
-        return batch
-
-    def _write_attachments(self, stream_entry: StreamEntry, uow: UnitOfWork):
-        service_kwargs = {}
-        if uow:
-            service_kwargs["uow"] = uow
-        entry_id = self._get_stream_entry_id(stream_entry)
-
-        existing_files = self._file_service.list_files(self._identity, entry_id)
-        existing_files = {f["key"]: f for f in existing_files.entries}
-
-        for f in stream_entry.files:
-            if f.metadata["key"] in existing_files:
-                if not self._update:
-                    continue
-                # TODO: compare if the file should be deleted and re-created
-                # if so, delete the file and create again
-                self._file_service.delete_file(
-                    self._identity, entry_id, f.metadata["key"], **service_kwargs
-                )
-
-            self._file_service.init_files(
-                self._identity,
-                entry_id,
-                [{"key": f.metadata["key"]}],
-                **service_kwargs,
-            )
-            metadata = f.metadata.get("metadata", {})
-            if metadata:
-                self._file_service.update_file_metadata(
-                    self._identity,
-                    entry_id,
-                    file_key=f.metadata["key"],
-                    data=metadata,
-                    **service_kwargs,
-                )
-            self._file_service.set_file_content(
-                self._identity,
-                entry_id,
-                f.metadata["key"],
-                BytesIO(attachments_requests.get(f.content_url).content),
-                **service_kwargs,
-            )
-            self._file_service.commit_file(
-                self._identity, entry_id, f.metadata["key"], **service_kwargs
-            )
-        new_files_keys = set(f.metadata["key"] for f in stream_entry.files)
-
-        for existing_file_key in existing_files:
-            if existing_file_key not in new_files_keys:
-                self._file_service.delete_file(
-                    self._identity, entry_id, existing_file_key, **service_kwargs
-                )