oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/datastreams/synchronous.py

@@ -1,70 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import logging
-from typing import List
-
-from ..proxies import current_datastreams
-from .datastreams import AbstractDataStream, DataStreamChain
-from .transformers import BaseTransformer
-from .types import DataStreamCallback, StreamEntryError
-from .writers import BaseWriter
-
-log = logging.getLogger("datastreams")
-
-
-class SynchronousDataStreamChain(DataStreamChain):
-    def __init__(self, transformers: List[BaseTransformer], writers: List[BaseWriter]):
-        self._transformers = transformers
-        self._writers = writers
-
-    def process(self, batch, callback: DataStreamCallback):
-        callback.batch_started(batch)
-        for transformer in self._transformers:
-            try:
-                batch = transformer.apply(batch) or batch
-            except Exception as ex:
-                if log.getEffectiveLevel():
-                    log.error(
-                        "Unexpected error in transformer: %s: %s",
-                        repr(transformer),
-                        repr(batch),
-                    )
-                batch.errors.append(StreamEntryError.from_exception(ex))
-                callback.transformer_error(batch, transformer, exception=ex)
-
-        for writer in self._writers:
-            try:
-                batch = writer.write(batch) or batch
-            except Exception as ex:
-                if log.getEffectiveLevel():
-                    log.error(
-                        "Unexpected error in writer: %s: %s", repr(writer), repr(batch)
-                    )
-                batch.errors.append(StreamEntryError.from_exception(ex))
-                callback.writer_error(batch, writer, exception=ex)
-        callback.batch_finished(batch)
-
-    def finish(self, callback: DataStreamCallback):
-        for writer in self._writers:
-            try:
-                writer.finish()
-            except Exception as e:
-                log.error("Unexpected error in writer: %s", repr(writer))
-                callback.writer_error(batch=None, writer=writer, exception=e)
-
-
-class SynchronousDataStream(AbstractDataStream):
-    """Data stream."""
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return SynchronousDataStreamChain(
-            transformers=[
-                current_datastreams.get_transformer(tr, identity=identity)
-                for tr in self._transformers
-            ],
-            writers=[
-                current_datastreams.get_writer(wr, identity=identity)
-                for wr in self._writers
-            ],
-        )

oarepo_runtime/datastreams/transformers.py

@@ -1,18 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Union
-
-from oarepo_runtime.datastreams.types import StreamBatch
-
-
-class BaseTransformer(ABC):
-    """Base transformer."""
-
-    def __init__(self, **kwargs) -> None:
-        pass
-
-    @abstractmethod
-    def apply(self, batch: StreamBatch, *args, **kwargs) -> Union[StreamBatch, None]:
-        """Applies the transformation to the entry.
-        :returns: A StreamEntry. The transformed entry.
-                  Raises TransformerError in case of errors.
-        """

oarepo_runtime/datastreams/types.py

@@ -1,323 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import dataclasses
-import json
-import logging
-import textwrap
-import traceback
-from typing import Any, Dict, List, Optional, Union
-
-from .errors import DataStreamError
-from .json import JSONObject
-
-log = logging.getLogger("datastreams")
-
-
-@dataclasses.dataclass
-class StreamEntryError:
-    code: str
-    message: str
-    location: Optional[str] = None
-    info: Union[JSONObject, None] = None
-
-    @classmethod
-    def from_exception(
-        cls, exc: Exception, limit=30, message=None, location=None, info=None, code=None
-    ):
-        if isinstance(exc, DataStreamError):
-            return cls(
-                code=exc.code,
-                message=exc.message,
-                location=exc.location,
-                info=exc.detail,
-            )
-
-        # can not use format_exception here as the signature is different for python 3.9 and python 3.10
-        stack = traceback.format_exc(limit=limit)
-        if message:
-            formatted_exception = message
-        elif hasattr(exc, "format_exception"):
-            formatted_exception = exc.format_exception()
-        else:
-            formatted_exception = str(exc)
-
-        return cls(
-            code=code or getattr(exc, "type", type(exc).__name__),
-            message=formatted_exception,
-            location=location,
-            info={
-                "message": str(exc),
-                "exception": type(exc).__name__,
-                "stack": stack,
-                **(info or {}),
-            },
-        )
-
-    @property
-    def json(self) -> JSONObject:
-        ret = {}
-        if self.code:
-            ret["code"] = self.code
-        if self.message:
-            ret["message"] = self.message
-        if self.location:
-            ret["location"] = self.location
-        if self.info:
-            ret["info"] = self.info
-        return ret
-
-    @classmethod
-    def from_json(cls, js: JSONObject):
-        if js is None:
-            return None
-        return cls(
-            code=js.get("code"),
-            message=js.get("message"),
-            location=js.get("location"),
-            info=js.get("info"),
-        )
-
-    def __str__(self):
-        formatted_info = textwrap.indent(
-            json.dumps(self.info or {}, ensure_ascii=False, indent=4), prefix="    "
-        )
-        return f"{self.code}:{self.location if self.location else ''} {self.message}\n{formatted_info}"
-
-    def __repr__(self):
-        return str(self)
-
-
-@dataclasses.dataclass
-class StreamEntryFile:
-    metadata: JSONObject
-    content_url: str
-    "data url with the content of the file or any other resolvable url"
-
-    @property
-    def json(self) -> JSONObject:
-        return {
-            "metadata": self.metadata,
-            "content_url": self.content_url,
-        }
-
-    @classmethod
-    def from_json(cls, js: JSONObject):
-        return cls(
-            metadata=js["metadata"],
-            content_url=js["content_url"],
-        )
-
-
-@dataclasses.dataclass
-class StreamEntry:
-    """Object to encapsulate streams processing."""
-
-    entry: JSONObject
-    files: List[StreamEntryFile] = dataclasses.field(default_factory=list)
-    seq: int = 0
-    id: Optional[str] = None
-    filtered: bool = False
-    deleted: bool = False
-    errors: List[StreamEntryError] = dataclasses.field(default_factory=list)
-    context: JSONObject = dataclasses.field(default_factory=dict)
-
-    @property
-    def ok(self):
-        return not self.filtered and not self.errors
-
-    @property
-    def json(self) -> JSONObject:
-        return {
-            "id": self.id,
-            "entry": self.entry,
-            "filtered": self.filtered,
-            "deleted": self.deleted,
-            "errors": [x.json for x in self.errors],
-            "context": self.context,
-            "seq": self.seq,
-            "files": [x.json for x in self.files],
-        }
-
-    @classmethod
-    def from_json(cls, js):
-        return cls(
-            id=js["id"],
-            entry=js["entry"],
-            filtered=js["filtered"],
-            deleted=js["deleted"],
-            errors=[StreamEntryError.from_json(x) for x in js["errors"]],
-            context=js["context"],
-            seq=js["seq"],
-            files=[StreamEntryFile.from_json(x) for x in js["files"]],
-        )
-
-    def __str__(self):
-        ret = [
-            f"Entry #{self.seq}: id {self.id or 'not yet set'}, filtered: {self.filtered}, deleted: {self.deleted}",
-            "Content:",
-            textwrap.indent(
-                json.dumps(self.entry, ensure_ascii=False, indent=4), "    "
-            ),
-            "Context:",
-            textwrap.indent(
-                json.dumps(self.context, ensure_ascii=False, indent=4), "    "
-            ),
-        ]
-        if self.errors:
-            ret.append("Errors:")
-            for error in self.errors:
-                ret.append(textwrap.indent(str(error), "    "))
-        return "\n".join(ret)
-
-
-@dataclasses.dataclass
-class StreamBatch:
-    entries: List[StreamEntry]
-    context: Dict[str, Any] = dataclasses.field(default_factory=dict)
-    last: bool = False
-    seq: int = 0
-    errors: List[StreamEntryError] = dataclasses.field(default_factory=list)
-
-    @property
-    def ok_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.ok]
-
-    @property
-    def failed_entries(self):
-        if self.errors:
-            return self.entries
-        return [x for x in self.entries if x.errors]
-
-    @property
-    def skipped_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.filtered]
-
-    @property
-    def deleted_entries(self):
-        if self.errors:
-            return []
-        return [x for x in self.entries if x.deleted]
-
-    @property
-    def json(self):
-        return {
-            "entries": [x.json for x in self.entries],
-            "context": self.context,
-            "last": self.last,
-            "seq": self.seq,
-            "errors": [x.json for x in self.errors],
-        }
-
-    @classmethod
-    def from_json(cls, js):
-        if js is None:
-            return None
-        try:
-            [StreamEntry.from_json(x) for x in js["entries"]]
-        except:
-            log.exception("Exception parsing %s", js)
-            raise
-        return cls(
-            entries=[StreamEntry.from_json(x) for x in js["entries"]],
-            context=js["context"],
-            last=js["last"],
-            seq=js["seq"],
-            errors=[StreamEntryError.from_json(x) for x in js["errors"]],
-        )
-
-
-class DataStreamCallback:
-    def __init__(self, log_error_entry=False):
-        self.log_error_entry = log_error_entry
-
-    def batch_started(self, batch):
-        log.info("Batch started: %s", batch.seq)
-        if log.isEnabledFor(logging.DEBUG):
-            log.debug("Content: %s", batch)
-
-    def batch_finished(self, batch: StreamBatch):
-        log.info("Batch finished: %s", batch.seq)
-        if log.isEnabledFor(logging.DEBUG):
-            log.debug("Content: %s", batch)
-        for err in batch.errors:
-            log.error("Failed batch: %s: %s", err, batch.seq)
-        if self.log_error_entry:
-            for entry in batch.entries:
-                if entry.errors:
-                    log.error("Failed entry: %s in batch %s", entry, batch.seq)
-
-    def reader_error(self, reader, exception):
-        log.error("Reader error: %s: %s", reader, exception)
-
-    def transformer_error(self, batch, transformer, exception):
-        log.error("Transformer error: %s: %s", transformer, exception)
-
-    def writer_error(self, batch, writer, exception):
-        log.error("Writer error: %s: %s", writer, exception)
-
-
-class StatsKeepingDataStreamCallback(DataStreamCallback):
-    def __init__(self, log_error_entry=False):
-        super().__init__(log_error_entry=log_error_entry)
-
-        self.started_batches_count = 0
-        self.finished_batches_count = 0
-        self.ok_entries_count = 0
-        self.filtered_entries_count = 0
-        self.deleted_entries_count = 0
-        self.failed_entries_count = 0
-        self.reader_errors_count = 0
-        self.transformer_errors_count = 0
-        self.writer_errors_count = 0
-
-    def batch_started(self, batch):
-        super().batch_started(batch)
-        self.started_batches_count += 1
-
-    def batch_finished(self, batch: StreamBatch):
-        super().batch_finished(batch)
-        self.finished_batches_count += 1
-        for entry in batch.entries:
-            if entry.ok:
-                self.ok_entries_count += 1
-            if entry.filtered:
-                self.filtered_entries_count += 1
-            if entry.deleted:
-                self.deleted_entries_count += 1
-            if entry.errors:
-                self.failed_entries_count += 1
-
-    def reader_error(self, reader, exception):
-        super().reader_error(reader, exception)
-        self.reader_errors_count += 1
-
-    def transformer_error(self, batch, transformer, exception):
-        super().transformer_error(batch, transformer, exception)
-        self.transformer_errors_count += 1
-
-    def writer_error(self, batch, writer, exception):
-        super().writer_error(batch, writer, exception)
-        self.writer_errors_count += 1
-
-    def stats(self):
-        ret = [f"{self.finished_batches_count} batches finished"]
-        if self.ok_entries_count:
-            ret.append(f"ok: {self.ok_entries_count}")
-        if self.deleted_entries_count:
-            ret.append(f"deleted: {self.deleted_entries_count}")
-        if self.filtered_entries_count:
-            ret.append(f"filtered: {self.filtered_entries_count}")
-        if self.failed_entries_count:
-            ret.append(f"failed: {self.failed_entries_count}")
-        if self.reader_errors_count:
-            ret.append(f"reader errors: {self.reader_errors_count}")
-        if self.transformer_errors_count:
-            ret.append(f"transformer errors: {self.transformer_errors_count}")
-        if self.writer_errors_count:
-            ret.append(f"writer errors: {self.writer_errors_count}")
-        return ", ".join(ret)

oarepo_runtime/datastreams/utils.py

@@ -1,131 +0,0 @@
-from base64 import b64decode
-
-import requests
-from deprecated import deprecated
-from flask import current_app
-from invenio_drafts_resources.services import RecordService as DraftRecordService
-from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services import FileService, RecordService
-from invenio_records_resources.services.records.results import RecordItem
-from requests import PreparedRequest, Response
-from requests.adapters import BaseAdapter
-
-
-def get_record_service_for_record(record):
-    if not record:
-        return None
-    if "OAREPO_PRIMARY_RECORD_SERVICE" in current_app.config:
-        return get_record_service_for_record_class(type(record))
-    else:
-        return get_record_service_for_record_deprecated(record)
-
-
-def get_record_service_for_record_class(record_cls):
-    service_id = current_app.config["OAREPO_PRIMARY_RECORD_SERVICE"][record_cls]
-    return current_service_registry.get(service_id)
-
-
-@deprecated(
-    version="1.5.43", reason="Please recompile model to remove this deprecation warning"
-)
-def get_record_service_for_record_deprecated(record):
-    if getattr(record, "is_draft", False):
-        record_name = "draft_cls"
-        expect_draft_service = True
-    else:
-        record_name = "record_cls"
-        expect_draft_service = False
-
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, RecordService):
-            continue
-        if isinstance(svc, FileService):
-            continue
-        is_draft_service = isinstance(svc, DraftRecordService)
-        if is_draft_service != expect_draft_service:
-            continue
-        service_record = getattr(svc, record_name, None)
-        if service_record == type(record):
-            return svc
-
-def get_file_service_for_record_class(record_class):
-    if not record_class:
-        return None
-
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, FileService):
-            continue
-        if svc.record_cls != record_class:
-            continue
-        return svc
-
-def get_file_service_for_file_record_class(file_record_class):
-    record_class = file_record_class.record_cls
-    return get_file_service_for_record_class(record_class)
-
-def get_file_service_for_record_service(
-    record_service, check_draft_files=True, record=None
-):
-    if isinstance(record, RecordItem):
-        record = record._record
-    if record and getattr(record, "is_draft", False) is False:
-        check_draft_files = False
-    if (
-        check_draft_files
-        and hasattr(record_service, "draft_files")
-        and isinstance(record_service.draft_files, FileService)
-    ):
-        return record_service.draft_files
-    if hasattr(record_service, "files") and isinstance(
-        record_service.files, FileService
-    ):
-        return record_service.files
-    return get_file_service_for_record_class(
-        getattr(record_service.config, "record_cls", None)
-    )
-
-
-def get_record_service_for_file_service(file_service, record=None):
-    if record and getattr(record, "is_draft", False):
-        record_name = "draft_cls"
-        expect_draft_service = True
-    else:
-        record_name = "record_cls"
-        expect_draft_service = False
-    for svc in current_service_registry._services.values():
-        if not isinstance(svc, RecordService):
-            continue
-        is_draft_service = isinstance(svc, DraftRecordService)
-        if is_draft_service != expect_draft_service:
-            continue
-        service_record = getattr(svc, record_name, None)
-        if service_record == file_service.record_cls:
-            return svc
-
-    raise KeyError(
-        f"Could not get service for file service {file_service}, draft {expect_draft_service}"
-    )
-
-
-class DataAdapter(BaseAdapter):
-    def send(
-        self,
-        request: PreparedRequest,
-        stream=False,
-        timeout=None,
-        verify=True,
-        cert=None,
-        proxies=None,
-    ):
-        data = request.url.replace("data:", "")
-        resp = Response()
-        resp.status_code = 200
-        resp._content = b64decode(data)
-        return resp
-
-    def close(self):
-        pass
-
-
-attachments_requests = requests.Session()
-attachments_requests.mount("data:", DataAdapter())

oarepo_runtime/datastreams/writers/__init__.py

@@ -1,21 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Union
-
-from oarepo_runtime.datastreams.types import StreamBatch
-
-
-class BaseWriter(ABC):
-    """Base writer."""
-
-    def __init__(self, **kwargs) -> None:
-        """kwargs for extensions"""
-
-    @abstractmethod
-    def write(self, batch: StreamBatch) -> Union[StreamBatch, None]:
-        """Writes the input entry to the target output.
-        :returns: nothing
-                  Raises WriterException in case of errors.
-        """
-
-    def finish(self):
-        pass

oarepo_runtime/datastreams/writers/attachments_file.py

@@ -1,92 +0,0 @@
-import os
-from pathlib import Path
-
-import yaml
-
-from oarepo_runtime.datastreams import StreamBatch, StreamEntry
-
-from ..utils import attachments_requests
-from . import BaseWriter
-
-
-class AttachmentsFileWriter(BaseWriter):
-    """
-    Writes the files and its metadata into subdirectories.
-
-    The path will be files/<record-id>/<file-id>/metadata.yaml for technical metadata
-    and files/<record-id>/<file-id>/<key> for the data.
-
-    If the data key is "metadata.yaml", then "metadata" will be placed to "metametadata.yaml"
-    """
-
-    def __init__(self, *, target, base_path=None, **kwargs):
-        """Constructor.
-        :param file_or_path: path of the output file.
-        """
-        super().__init__(**kwargs)
-        self._grouping = 3
-        self._min_padding = 3
-        if base_path:
-            self._dir = Path(base_path).joinpath(target)
-        else:
-            self._dir = Path(target)
-
-    def write(self, batch: StreamBatch, *args, **kwargs):
-        """Writes the input stream entry using a given service."""
-        """
-        context looks like: {
-            'files': [
-            {'metadata': {'updated': '...', 'mimetype': 'image/png', 'storage_class': 'L', 'file_id': '', 
-                          'links': {...}, 'size': 27, 'status': 'completed', 'version_id': '...', 
-                          'bucket_id': '...', 'metadata': None, 'key': 'test.png', 
-                          'checksum': 'md5:...', 'created': '...'}, 
-             'content': b'test file content: test.png'}]}
-        """
-        for entry in batch.entries:
-            if entry.ok and entry.files:
-                self.write_entry(entry)
-
-    def write_entry(self, entry: StreamEntry):
-        dirname = self._dir.joinpath(format_serial(entry.seq)) / "data"
-        dirname.mkdir(parents=True, exist_ok=False)
-        file_keys = []
-        files_metadata = []
-        for fn_idx, fn in enumerate(entry.files):
-            md = {**fn.metadata}
-            content = attachments_requests.get(fn.content_url).content
-            # cleanup
-            md.pop("storage_class", None)
-            md.pop("file_id", None)
-            md.pop("links", None)
-            md.pop("status", None)
-            md.pop("version_id", None)
-            md.pop("bucket_id", None)
-            key = md["key"]
-            file_keys.append(key)
-            files_metadata.append(md)
-            (dirname / key).write_bytes(content)
-        metadata_key = "metadata.yaml"
-        while metadata_key in file_keys:
-            metadata_key = "meta_" + metadata_key
-        with open(dirname / metadata_key, "w") as f:
-            yaml.safe_dump_all(files_metadata, f)
-        return entry
-
-    def finish(self):
-        """Finalizes writing"""
-
-
-def format_serial(serial_no):
-    grouping = 3
-    min_padding = 3
-    serial_no = str(serial_no)
-    formatted_length = max(min_padding, len(serial_no))
-    while formatted_length % grouping:
-        formatted_length += 1
-    padded_serial = serial_no.zfill(formatted_length)
-    return os.sep.join(
-        [
-            padded_serial[i : i + grouping]
-            for i in range(0, len(padded_serial), grouping)
-        ]
-    )