oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/datastreams/asynchronous.py

@@ -1,247 +0,0 @@
-import logging
-from typing import Any, Dict, List, Union
-
-import celery
-from celery.canvas import Signature as CelerySignature
-from celery.canvas import chain
-from celery.result import allow_join_result
-from flask_principal import (
-    ActionNeed,
-    Identity,
-    ItemNeed,
-    Need,
-    RoleNeed,
-    TypeNeed,
-    UserNeed,
-)
-
-from oarepo_runtime.datastreams.datastreams import (
-    AbstractDataStream,
-    DataStreamChain,
-    Signature,
-)
-
-from .datastreams import DataStreamCallback, StreamBatch
-from .json import JSONObject
-from .types import StreamEntryError
-from .writers import BaseWriter
-
-timing = logging.getLogger("oai.harvester.timing")
-log = logging.getLogger("datastreams")
-
-
-class AsynchronousDataStream(AbstractDataStream):
-    def __init__(
-        self,
-        *,
-        readers: List[Union[Signature, Any]],
-        writers: List[Union[Signature, Any]],
-        transformers: List[Union[Signature, Any]] = None,
-        callback: Union[DataStreamCallback, Any],
-        batch_size=100,
-        on_background=True,
-        reader_callback=None,
-    ):
-        super().__init__(
-            readers=readers,
-            writers=writers,
-            transformers=transformers,
-            callback=callback,
-            batch_size=batch_size,
-            reader_callback=reader_callback,
-        )
-        self._on_background = on_background
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return AsynchronousDataStreamChain(
-            transformers=self._transformers,
-            writers=self._writers,
-            on_background=self._on_background,
-            identity=identity,
-        )
-
-    def _reader_error(self, reader, exception):
-        self._callback.apply(
-            kwargs={
-                "callback": f"reader_error",
-                "exception": StreamEntryError.from_exception(exception).json,
-            }
-        )
-
-
-class AsynchronousDataStreamChain(DataStreamChain):
-    def __init__(
-        self,
-        identity: Identity,
-        transformers: List[Signature],
-        writers: List[Signature],
-        on_background=True,
-    ):
-        self._transformers = transformers
-        self._writers = writers
-        self._on_background = on_background
-        self._identity = identity
-
-    def process(self, batch: StreamBatch, callback: CelerySignature):
-        chain = self._prepare_chain(callback)
-        self._call(chain, batch=batch.json)
-
-    def _prepare_chain(self, callback: CelerySignature):
-        chain_def = [
-            datastreams_call_callback.signature(
-                (), kwargs={"callback": callback, "callback_name": "batch_started"}
-            )
-        ]
-        serialized_identity = serialize_identity(self._identity)
-        if self._transformers:
-            for transformer in self._transformers:
-                chain_def.append(
-                    run_datastream_processor.signature(
-                        kwargs={
-                            "processor": transformer.json,
-                            "identity": serialized_identity,
-                            "callback": callback,
-                        }
-                    )
-                )
-
-        for writer in self._writers:
-            chain_def.append(
-                run_datastream_processor.signature(
-                    kwargs={
-                        "processor": writer.json,
-                        "identity": serialized_identity,
-                        "callback": callback,
-                    }
-                )
-            )
-
-        chain_def.append(
-            datastreams_call_callback.signature(
-                (),
-                kwargs=dict(
-                    callback=callback,
-                    callback_name="batch_finished",
-                    identity=serialized_identity,
-                ),
-            )
-        )
-
-        chain_sig = chain(*chain_def)
-        chain_sig.link_error(
-            datastreams_error_callback.signature(
-                (),
-                kwargs=dict(
-                    callback=callback,
-                    callback_name="error",
-                    identity=serialized_identity,
-                ),
-            )
-        )
-        return chain_sig
-
-    def _call(self, sig, **kwargs):
-        if self._on_background:
-            call = sig.apply_async
-        else:
-            call = sig.apply
-        call([], kwargs)
-
-    def finish(self, callback: Signature):
-        "nothing to finish here, dumpers needing finish (such as file dumpers) are not supported in async"
-
-
-@celery.shared_task
-def run_datastream_processor(batch: Dict, *, processor: JSONObject, identity, callback):
-    identity = deserialize_identity(identity)
-    processor_signature = Signature.from_json(processor)
-    deserialized_batch: StreamBatch = StreamBatch.from_json(batch)
-
-    processor = processor_signature.resolve(identity=identity)
-    try:
-        if isinstance(processor, BaseWriter):
-            deserialized_batch = (
-                processor.write(deserialized_batch) or deserialized_batch
-            )
-        else:
-            deserialized_batch = (
-                processor.apply(deserialized_batch) or deserialized_batch
-            )
-
-    except Exception as ex:
-        log.exception("Error processing batch inside %s", processor_signature)
-
-        err = StreamEntryError.from_exception(ex)
-        deserialized_batch.errors.append(err)
-        callback.apply(
-            (),
-            {
-                "batch": deserialized_batch.json,
-                "identity": serialize_identity(identity),
-                "callback": f"{processor_signature.kind.value}_error",
-                "exception": err.json,
-            },
-        )
-    return deserialized_batch.json
-
-
-@celery.shared_task
-def datastreams_call_callback(
-    batch: Dict, *, identity=None, callback, callback_name, **kwargs
-):
-    callback = CelerySignature(callback)
-    callback.apply(
-        kwargs=dict(batch=batch, identity=identity, callback=callback_name, **kwargs)
-    )
-    return batch
-
-
-@celery.shared_task
-def datastreams_error_callback(
-    parent_task_id, *, identity=None, callback, callback_name, **kwargs
-):
-    with allow_join_result():
-        from celery import current_app
-
-        result = current_app.AsyncResult(parent_task_id)
-        result.get(propagate=False)
-
-        callback = CelerySignature(callback)
-        callback.apply(
-            kwargs=dict(
-                batch={},
-                identity=identity,
-                callback=callback_name,
-                result=result.result,
-                traceback=result.traceback,
-                **kwargs,
-            )
-        )
-
-
-def serialize_identity(identity):
-    return {
-        "id": identity.id,
-        "auth_type": identity.auth_type,
-        "provides": [
-            {"type": type(x).__name__, "params": x._asdict()} for x in identity.provides
-        ],
-    }
-
-
-def deserialize_identity(identity_dict):
-    if identity_dict is None:
-        return None
-    ret = Identity(id=identity_dict["id"], auth_type=identity_dict["auth_type"])
-    for provide in identity_dict["provides"]:
-        clz = {
-            "Need": Need,
-            "UserNeed": UserNeed,
-            "RoleNeed": RoleNeed,
-            "TypeNeed": TypeNeed,
-            "ActionNeed": ActionNeed,
-            "ItemNeed": ItemNeed,
-        }[provide["type"]]
-
-        ret.provides.add(clz(**provide["params"]))
-    return ret

oarepo_runtime/datastreams/catalogue.py

@@ -1,150 +0,0 @@
-import dataclasses
-from pathlib import Path
-from typing import Iterator, List
-
-import yaml
-from flask import current_app
-
-from oarepo_runtime.datastreams.datastreams import Signature, SignatureKind
-
-from .errors import DataStreamCatalogueError
-
-
-@dataclasses.dataclass
-class CatalogueDataStream:
-    stream_name: str
-    readers: List[Signature]
-    writers: List[Signature]
-    transformers: List[Signature]
-
-
-class DataStreamCatalogue:
-    def __init__(self, catalogue, content=None) -> None:
-        """
-        Catalogue of data streams. The catalogue contains a dict of:
-        stream_name: stream_definition, where stream definition is an array of:
-
-        - reader: reader_class
-          <rest of parameters go to reader constructor>
-        - transformer: transformer_class
-          <rest of parameters go to transformer constructor>
-        - writer: writer_class
-          <rest of parameters go to writer constructor>
-
-        If reader class is not passed and _source_ is, then the reader class will be taken from the
-        DATASTREAMS_READERS_BY_EXTENSION config variable - map from file extension to reader class.
-
-        If 'service' is passed, service writer will be used with this service
-
-        Transformer class must always be passed.
-        """
-        self._catalogue_path = Path(catalogue)
-        if content:
-            self._catalogue = content
-        else:
-            with open(catalogue) as f:
-                self._catalogue = yaml.safe_load(f)
-
-    @property
-    def path(self):
-        return self._catalogue_path
-
-    @property
-    def directory(self):
-        return self._catalogue_path.parent
-
-    def get_datastreams(self) -> Iterator[CatalogueDataStream]:
-        for stream_name in self._catalogue:
-            yield self.get_datastream(stream_name)
-
-    def __iter__(self):
-        return iter(self._catalogue)
-
-    def get_datastream(
-        self,
-        stream_name,
-    ) -> CatalogueDataStream:
-        stream_definition = self._catalogue[stream_name]
-        readers = []
-        transformers = []
-        writers = []
-        for entry in stream_definition:
-            entry = {**entry}
-            try:
-                if "reader" in entry:
-                    readers.append(
-                        get_signature(
-                            "reader",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "transformer" in entry:
-                    transformers.append(
-                        get_signature(
-                            "transformer",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "writer" in entry:
-                    writers.append(
-                        get_signature(
-                            "writer",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "source" in entry:
-                    readers.append(self.get_reader(entry))
-                elif "service" in entry:
-                    writers.append(self.get_service_writer(entry))
-                else:
-                    raise DataStreamCatalogueError(
-                        "Can not decide what this record is - reader, transformer or service?"
-                    )
-            except DataStreamCatalogueError as e:
-                e.entry = entry
-                e.stream_name = stream_name
-                raise e
-        return CatalogueDataStream(
-            stream_name=stream_name,
-            readers=readers,
-            transformers=transformers,
-            writers=writers,
-        )
-
-    def get_reader(self, entry):
-        entry = {**entry}
-        if not entry.get("reader"):
-            try:
-                source = Path(entry["source"])
-                ext = source.suffix[1:]
-                reader_class = (
-                    current_app.config["DATASTREAMS_READERS_BY_EXTENSION"].get(ext)
-                    or current_app.config["DEFAULT_DATASTREAMS_READERS_BY_EXTENSION"][
-                        ext
-                    ]
-                )
-                entry["reader"] = reader_class
-            except KeyError:
-                raise DataStreamCatalogueError(
-                    f"Do not have loader for file {source} - extension {ext} not defined in DATASTREAMS_READERS_BY_EXTENSION config"
-                )
-        return get_signature(
-            "reader",
-            entry,
-            base_path=str(self.directory),
-        )
-
-    def get_service_writer(self, entry):
-        return Signature(
-            SignatureKind("writer"),
-            "service",
-            kwargs={**entry, "base_path": str(self.directory)},
-        )
-
-
-def get_signature(kind, entry, **kwargs):
-    entry = {**entry, **kwargs}
-    return Signature(kind=SignatureKind(kind), name=entry.pop(kind), kwargs=entry)

oarepo_runtime/datastreams/datastreams.py

@@ -1,152 +0,0 @@
-import abc
-import copy
-import dataclasses
-from enum import Enum
-from typing import Any, Callable, Iterator, List, Union
-
-from invenio_access.permissions import system_identity
-
-from oarepo_runtime.datastreams.types import (
-    DataStreamCallback,
-    StreamBatch,
-    StreamEntry,
-)
-from oarepo_runtime.proxies import current_datastreams
-
-from .json import JSONObject
-
-
-class DataStreamChain(abc.ABC):
-    @abc.abstractmethod
-    def process(self, batch: StreamBatch, callback: Union[DataStreamCallback, Any]):
-        pass
-
-    @abc.abstractmethod
-    def finish(self, callback: Union[DataStreamCallback, Any]):
-        pass
-
-try:
-    from enum import StrEnum
-
-    class SignatureKind(StrEnum):
-        READER = "reader"
-        TRANSFORMER = "transformer"
-        WRITER = "writer"
-
-except ImportError:
-
-    class SignatureKind(str, Enum):
-        READER = "reader"
-        TRANSFORMER = "transformer"
-        WRITER = "writer"
-
-
-@dataclasses.dataclass
-class Signature:
-    kind: SignatureKind
-    name: str
-    kwargs: JSONObject
-
-    @property
-    def json(self):
-        return {"kind": self.kind.value, "name": self.name, "kwargs": self.kwargs}
-
-    @classmethod
-    def from_json(cls, json):
-        return cls(
-            kind=SignatureKind(json["kind"]),
-            name=json["name"],
-            kwargs=json["kwargs"],
-        )
-
-    def resolve(self, *, identity, **kwargs):
-        if self.kind == SignatureKind.TRANSFORMER:
-            return current_datastreams.get_transformer(
-                self, **kwargs, identity=identity
-            )
-        elif self.kind == SignatureKind.WRITER:
-            return current_datastreams.get_writer(self, **kwargs, identity=identity)
-        else:
-            raise ValueError(f"Unknown signature kind: {self.kind}")
-
-
-class AbstractDataStream(abc.ABC):
-    def __init__(
-        self,
-        *,
-        readers: List[Union[Signature, Any]],
-        writers: List[Union[Signature, Any]],
-        transformers: List[Union[Signature, Any]] = None,
-        callback: Union[DataStreamCallback, Signature],
-        batch_size=1,
-        identity=system_identity,
-        reader_callback: Callable[[StreamBatch], None] = None,
-    ):
-        """Constructor.
-        :param readers: an ordered list of readers (whatever a reader is).
-        :param writers: an ordered list of writers (whatever a writer is).
-        :param transformers: an ordered list of transformers to apply (whatever a transformer is).
-        """
-        self._readers: List[Signature] = [*readers]
-        self._transformers: List[Signature] = [*(transformers or [])]
-        self._writers: List[Signature] = [*writers]
-        self._callback = callback
-        self._batch_size = batch_size
-        self._identity = identity
-        self._reader_callback = reader_callback
-
-    def _read_entries(self) -> Iterator[StreamEntry]:
-        seq = 0
-        for reader_signature in self._readers:
-            reader = current_datastreams.get_reader(
-                reader_signature, identity=self._identity
-            )
-            try:
-                for entry in reader:
-                    seq += 1
-                    entry.seq = seq
-                    yield entry
-            except Exception as ex:
-                self._reader_error(reader, exception=ex)
-
-    def _read_batches(self, context) -> Iterator[StreamBatch]:
-        batch_entries = []
-        batch_number = 0
-
-        def batch_maker(last=False):
-            nonlocal batch_number, batch_entries
-            batch_number += 1
-            ret = StreamBatch(
-                entries=batch_entries,
-                seq=batch_number,
-                context=copy.deepcopy(context),
-                last=last,
-            )
-            batch_entries = []
-            return ret
-
-        for entry in self._read_entries():
-            if len(batch_entries) == self._batch_size:
-                batch = batch_maker()
-                if self._reader_callback:
-                    self._reader_callback(batch)
-                yield batch
-                batch_entries = []
-            batch_entries.append(entry)
-        batch = batch_maker(last=True)
-        if self._reader_callback:
-            self._reader_callback(batch)
-        yield batch
-
-    def process(self, context=None, identity=system_identity):
-        context = context or {}
-        chain = self.build_chain(identity)
-        for batch in self._read_batches(context):
-            chain.process(batch, self._callback)
-
-    @abc.abstractmethod
-    def build_chain(self, identity) -> DataStreamChain:
-        pass
-
-    def _reader_error(self, reader, exception):
-        self._callback.reader_error(reader, exception=exception)

oarepo_runtime/datastreams/errors.py

@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2021 CERN.
-#
-# Invenio-Vocabularies is free software; you can redistribute it and/or
-# modify it under the terms of the MIT License; see LICENSE file for more
-# details.
-
-"""Datastream errors."""
-from typing import Union
-
-from .json import JSONObject
-
-
-class DataStreamError(Exception):
-    def __init__(
-        self,
-        message,
-        code=None,
-        location=None,
-        detail: Union[JSONObject, None] = None,
-    ):
-        """
-        @param message: a string message (overview)
-        @param code: a machine processable code
-        @param location: location inside the json, where the error was detected. Using dot notation,
-               arrays are indexed from 0, for example: `metadata.titles.0.language`
-        @param detail: a json-serializable object (dictionary) with details
-        """
-        super().__init__(message)
-        assert detail is None or isinstance(detail, dict)
-        self.detail = detail
-        self.message = message
-        self.code = code
-        self.location = location
-
-
-class ReaderError(DataStreamError):
-    """Transformer application exception."""
-
-
-class TransformerError(DataStreamError):
-    """Transformer application exception."""
-
-
-class WriterError(DataStreamError):
-    """Transformer application exception."""
-
-
-class DataStreamCatalogueError(Exception):
-    def __init__(self, message, entry=None, stream_name=None) -> None:
-        super().__init__(message)
-        self.entry = entry
-        self.stream_name = stream_name

oarepo_runtime/datastreams/ext.py

@@ -1,41 +0,0 @@
-import functools
-
-from invenio_base.utils import obj_or_import_string
-
-from oarepo_runtime.datastreams.datastreams import Signature
-
-
-class OARepoDataStreamsExt:
-    def __init__(self, app):
-        self.app = app
-
-    def get_reader(self, reader, identity, **kwargs):
-        return self._get_instance("DATASTREAMS_READERS", identity, kwargs, reader)
-
-    def get_writer(self, writer, identity, **kwargs):
-        return self._get_instance("DATASTREAMS_WRITERS", identity, kwargs, writer)
-
-    def get_transformer(self, transformer, identity, **kwargs):
-        return self._get_instance(
-            "DATASTREAMS_TRANSFORMERS", identity, kwargs, transformer
-        )
-
-    def _get_instance(self, config_name, identity, kwargs, inst):
-        if isinstance(inst, Signature):
-            config_classes = self._get_classes_from_config(config_name)
-            if inst.name not in config_classes:
-                raise KeyError(f"'{inst.name}' not found in config {config_name}")
-            reader_class = config_classes[inst.name]
-            all_kwargs = {**(inst.kwargs or {}), **kwargs}
-            if "identity" not in all_kwargs:
-                all_kwargs["identity"] = identity
-            return reader_class(**all_kwargs)
-        else:
-            return inst
-
-    @functools.lru_cache(maxsize=5)
-    def _get_classes_from_config(self, config_name):
-        return {
-            class_key: obj_or_import_string(class_name)
-            for class_key, class_name in self.app.config[config_name].items()
-        }