oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime/datastreams/readers/excel.py

@@ -1,123 +0,0 @@
-import re
-from traceback import format_exc
-from typing import Iterator
-
-import openpyxl
-
-from ..errors import ReaderError
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class ExcelReader(AttachmentsReaderMixin, BaseReader):
-    def iter_entries(self) -> Iterator[StreamEntry]:
-        with self._open("rb") as f:
-            try:
-                wb_obj = openpyxl.load_workbook(f)
-                sheet_obj = wb_obj.active
-            except Exception as err:
-                raise ReaderError(
-                    f"Cannot decode excel file {self._data_file.name}: {str(err)}",
-                    code="EXCEL_DECODE_ERROR",
-                    detail={
-                        "message": str(err),
-                        "exception": type(err).__name__,
-                        "stack": format_exc(limit=10),
-                    },
-                )
-
-            header, data = self.get_excel_data(sheet_obj)
-            for row in data:
-                yield StreamEntry(row)
-
-    def get_excel_data(self, sheet_obj):
-        """
-        returns an iterator (header, data)
-        """
-        header = []
-        data = []
-        it = sheet_obj.iter_rows()
-
-        try:
-            row = next_row(it)
-            while empty(row):
-                row = next_row(it)
-            while not empty(row):
-                header.append(row)
-                row = next_row(it)
-            while empty(row):
-                row = next_row(it)
-            while True:
-                if not empty(row):
-                    data.append(row)
-                row = next_row(it)
-        except StopIteration:
-            pass
-        if not data:
-            return [], self.to_dict(header)
-        else:
-            return self.to_dict(header), self.to_dict(data)
-
-    def to_dict(self, dta):
-        def is_array(val):
-            try:
-                int(val)
-                return True
-            except:
-                return False
-
-        def set_single(container, key, val):
-            try:
-                key = int(key)
-                while key >= len(container):
-                    container.append(None)
-                container[key] = val
-            except (TypeError, ValueError):
-                container[key] = val
-
-        def iterset(k, v, container):
-            while True:
-                current_key = k[0]
-                next_key = k[1] if len(k) > 1 else None
-                if not next_key:
-                    set_single(container, current_key, v)
-                    return
-                if isinstance(container, list):
-                    container.append({} if not is_array(next_key) else [])
-                    container = container[-1]
-                else:
-                    container = container.setdefault(
-                        current_key, {} if not is_array(next_key) else []
-                    )
-                k = k[1:]
-
-        def to_dict_item(header, item):
-            ret = RowDict()
-            for k, v in zip(header, item):
-                if not k:
-                    continue
-                v = v if v is not None else ""
-                v = str(v).strip()
-                if v:
-                    iterset(k, v, ret)
-            return ret
-
-        keys = [re.split("[_.]", x) if x else None for x in dta[0]]
-        return [to_dict_item(keys, d) for d in dta[1:]]
-
-
-def next_row(it):
-    return [x.value for x in next(it)]
-
-
-def empty(r):
-    for val in r:
-        if val:
-            return False
-    return True
-
-
-class RowDict(dict):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._header = {}

oarepo_runtime/datastreams/readers/json.py

@@ -1,27 +0,0 @@
-import json
-from typing import Iterator
-
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class JSONReader(AttachmentsReaderMixin, BaseReader):
-    """JSON Lines data iterator that loads records from JSON Lines files."""
-
-    def iter_entries(self) -> Iterator[StreamEntry]:
-        """Iterate over records."""
-        with self._open() as fp:
-            data = json.load(fp)
-            assert isinstance(data, list)
-            for d in data:
-                yield StreamEntry(d)
-
-
-class JSONLinesReader(BaseReader):
-    """JSON Lines data iterator that loads records from JSON Lines files."""
-
-    def __iter__(self) -> Iterator[StreamEntry]:
-        """Iterate over records."""
-        with self._open() as fp:
-            for line in fp:
-                yield StreamEntry(json.loads(line))

oarepo_runtime/datastreams/readers/service.py

@@ -1,54 +0,0 @@
-from base64 import b64encode
-from typing import List
-
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-
-from ..types import StreamEntryFile
-from ..utils import get_file_service_for_record_class
-from . import BaseReader, StreamEntry
-
-
-class ServiceReader(BaseReader):
-    """Writes the entries to a repository instance using a Service object."""
-
-    def __init__(self, *, service=None, identity=None, load_files=False, **kwargs):
-        """Constructor.
-        :param service_or_name: a service instance or a key of the
-                                service registry.
-        :param identity: access identity.
-        :param update: if True it will update records if they exist.
-        """
-        super().__init__(**kwargs)
-
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        self._service = service
-        self._identity = identity or system_identity
-        self._file_service = None
-        self._record_cls = getattr(self._service.config, "record_cls", None)
-
-        if self._record_cls and load_files:
-            # try to get file service
-            self._file_service = get_file_service_for_record_class(self._record_cls)
-
-    def __iter__(self):
-        for idx, entry in enumerate(self._service.scan(self._identity)):
-            files: List[StreamEntryFile] = []
-            if self._file_service:
-                for f in self._file_service.list_files(
-                    self._identity, entry["id"]
-                ).entries:
-                    file_item = self._file_service.get_file_content(
-                        self._identity, entry["id"], f["key"]
-                    )
-                    with file_item.open_stream("rb") as ff:
-                        base64_content = b64encode(ff.read()).decode("ascii")
-                        files.append(
-                            StreamEntryFile(
-                                metadata=f, content_url=f"data:{base64_content}"
-                            )
-                        )
-
-            yield StreamEntry(entry, files=files)

oarepo_runtime/datastreams/readers/yaml.py

@@ -1,14 +0,0 @@
-import yaml
-
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class YamlReader(AttachmentsReaderMixin, BaseReader):
-    """YAML data iterator that loads records from YAML files."""
-
-    def iter_entries(self):
-        """Iterate over records."""
-        with self._open() as fp:
-            for entry in yaml.safe_load_all(fp):
-                yield StreamEntry(entry)

oarepo_runtime/datastreams/semi_asynchronous.py

@@ -1,91 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import logging
-from typing import List
-
-from celery import shared_task
-from celery.canvas import Signature as CelerySignature
-
-from .asynchronous import (
-    AsynchronousDataStream,
-    AsynchronousDataStreamChain,
-    deserialize_identity,
-    serialize_identity,
-)
-from .datastreams import DataStreamChain, Signature
-from .transformers import BaseTransformer
-from .types import JSONObject, StreamBatch, StreamEntryError
-
-log = logging.getLogger("datastreams")
-
-
-class SemiAsynchronousDataStreamChain(AsynchronousDataStreamChain):
-    def _prepare_chain(self, callback: CelerySignature):
-        serialized_identity = serialize_identity(self._identity)
-        return run_semi_asynchronous_datastream_processor.s(
-            transformers=[tr.json for tr in self._transformers],
-            writers=[wr.json for wr in self._writers],
-            identity=serialized_identity,
-            callback=callback,
-        )
-
-
-class SemiAsynchronousDataStream(AsynchronousDataStream):
-    """Data stream."""
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return SemiAsynchronousDataStreamChain(
-            transformers=self._transformers,
-            writers=self._writers,
-            on_background=self._on_background,
-            identity=identity,
-        )
-
-
-@shared_task
-def run_semi_asynchronous_datastream_processor(
-    batch: JSONObject,
-    *,
-    transformers: List[JSONObject],
-    writers: List[JSONObject],
-    identity: JSONObject,
-    callback: CelerySignature,
-):
-    """Run datastream processor."""
-
-    callback.apply(kwargs={"callback": "batch_started", "batch": batch})
-
-    batch = StreamBatch.from_json(batch)
-    identity = deserialize_identity(identity)
-
-    for signature in (transformers or []) + (writers or []):
-        signature = Signature.from_json(signature)
-        try:
-            processor = signature.resolve(identity=identity)
-            if isinstance(processor, BaseTransformer):
-                batch = processor.apply(batch) or batch
-            else:
-                batch = processor.write(batch) or batch
-        except Exception as ex:
-            if log.getEffectiveLevel():
-                log.error(
-                    "Unexpected error in %s: %s",
-                    repr(signature),
-                    repr(batch),
-                )
-            err = StreamEntryError.from_exception(ex)
-            batch.errors.append(err)
-            callback.apply(
-                (),
-                {
-                    "batch": batch.json,
-                    "identity": serialize_identity(identity),
-                    "callback": f"{signature.kind.value}_error",
-                    "exception": err.json,
-                },
-            )
-
-    callback.apply(kwargs={"callback": "batch_finished", "batch": batch.json})
-
-    return None  # do not return anything to avoid redis pollution

oarepo_runtime/datastreams/synchronous.py

@@ -1,70 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import logging
-from typing import List
-
-from ..proxies import current_datastreams
-from .datastreams import AbstractDataStream, DataStreamChain
-from .transformers import BaseTransformer
-from .types import DataStreamCallback, StreamEntryError
-from .writers import BaseWriter
-
-log = logging.getLogger("datastreams")
-
-
-class SynchronousDataStreamChain(DataStreamChain):
-    def __init__(self, transformers: List[BaseTransformer], writers: List[BaseWriter]):
-        self._transformers = transformers
-        self._writers = writers
-
-    def process(self, batch, callback: DataStreamCallback):
-        callback.batch_started(batch)
-        for transformer in self._transformers:
-            try:
-                batch = transformer.apply(batch) or batch
-            except Exception as ex:
-                if log.getEffectiveLevel():
-                    log.error(
-                        "Unexpected error in transformer: %s: %s",
-                        repr(transformer),
-                        repr(batch),
-                    )
-                batch.errors.append(StreamEntryError.from_exception(ex))
-                callback.transformer_error(batch, transformer, exception=ex)
-
-        for writer in self._writers:
-            try:
-                batch = writer.write(batch) or batch
-            except Exception as ex:
-                if log.getEffectiveLevel():
-                    log.error(
-                        "Unexpected error in writer: %s: %s", repr(writer), repr(batch)
-                    )
-                batch.errors.append(StreamEntryError.from_exception(ex))
-                callback.writer_error(batch, writer, exception=ex)
-        callback.batch_finished(batch)
-
-    def finish(self, callback: DataStreamCallback):
-        for writer in self._writers:
-            try:
-                writer.finish()
-            except Exception as e:
-                log.error("Unexpected error in writer: %s", repr(writer))
-                callback.writer_error(batch=None, writer=writer, exception=e)
-
-
-class SynchronousDataStream(AbstractDataStream):
-    """Data stream."""
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return SynchronousDataStreamChain(
-            transformers=[
-                current_datastreams.get_transformer(tr, identity=identity)
-                for tr in self._transformers
-            ],
-            writers=[
-                current_datastreams.get_writer(wr, identity=identity)
-                for wr in self._writers
-            ],
-        )

oarepo_runtime/datastreams/transformers.py

@@ -1,18 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Union
-
-from oarepo_runtime.datastreams.types import StreamBatch
-
-
-class BaseTransformer(ABC):
-    """Base transformer."""
-
-    def __init__(self, **kwargs) -> None:
-        pass
-
-    @abstractmethod
-    def apply(self, batch: StreamBatch, *args, **kwargs) -> Union[StreamBatch, None]:
-        """Applies the transformation to the entry.
-        :returns: A StreamEntry. The transformed entry.
-                  Raises TransformerError in case of errors.
-        """