oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime/datastreams/datastreams.py

@@ -1,152 +0,0 @@
-import abc
-import copy
-import dataclasses
-from enum import Enum
-from typing import Any, Callable, Iterator, List, Union
-
-from invenio_access.permissions import system_identity
-
-from oarepo_runtime.datastreams.types import (
-    DataStreamCallback,
-    StreamBatch,
-    StreamEntry,
-)
-from oarepo_runtime.proxies import current_datastreams
-
-from .json import JSONObject
-
-
-class DataStreamChain(abc.ABC):
-    @abc.abstractmethod
-    def process(self, batch: StreamBatch, callback: Union[DataStreamCallback, Any]):
-        pass
-
-    @abc.abstractmethod
-    def finish(self, callback: Union[DataStreamCallback, Any]):
-        pass
-
-try:
-    from enum import StrEnum
-
-    class SignatureKind(StrEnum):
-        READER = "reader"
-        TRANSFORMER = "transformer"
-        WRITER = "writer"
-
-except ImportError:
-
-    class SignatureKind(str, Enum):
-        READER = "reader"
-        TRANSFORMER = "transformer"
-        WRITER = "writer"
-
-
-@dataclasses.dataclass
-class Signature:
-    kind: SignatureKind
-    name: str
-    kwargs: JSONObject
-
-    @property
-    def json(self):
-        return {"kind": self.kind.value, "name": self.name, "kwargs": self.kwargs}
-
-    @classmethod
-    def from_json(cls, json):
-        return cls(
-            kind=SignatureKind(json["kind"]),
-            name=json["name"],
-            kwargs=json["kwargs"],
-        )
-
-    def resolve(self, *, identity, **kwargs):
-        if self.kind == SignatureKind.TRANSFORMER:
-            return current_datastreams.get_transformer(
-                self, **kwargs, identity=identity
-            )
-        elif self.kind == SignatureKind.WRITER:
-            return current_datastreams.get_writer(self, **kwargs, identity=identity)
-        else:
-            raise ValueError(f"Unknown signature kind: {self.kind}")
-
-
-class AbstractDataStream(abc.ABC):
-    def __init__(
-        self,
-        *,
-        readers: List[Union[Signature, Any]],
-        writers: List[Union[Signature, Any]],
-        transformers: List[Union[Signature, Any]] = None,
-        callback: Union[DataStreamCallback, Signature],
-        batch_size=1,
-        identity=system_identity,
-        reader_callback: Callable[[StreamBatch], None] = None,
-    ):
-        """Constructor.
-        :param readers: an ordered list of readers (whatever a reader is).
-        :param writers: an ordered list of writers (whatever a writer is).
-        :param transformers: an ordered list of transformers to apply (whatever a transformer is).
-        """
-        self._readers: List[Signature] = [*readers]
-        self._transformers: List[Signature] = [*(transformers or [])]
-        self._writers: List[Signature] = [*writers]
-        self._callback = callback
-        self._batch_size = batch_size
-        self._identity = identity
-        self._reader_callback = reader_callback
-
-    def _read_entries(self) -> Iterator[StreamEntry]:
-        seq = 0
-        for reader_signature in self._readers:
-            reader = current_datastreams.get_reader(
-                reader_signature, identity=self._identity
-            )
-            try:
-                for entry in reader:
-                    seq += 1
-                    entry.seq = seq
-                    yield entry
-            except Exception as ex:
-                self._reader_error(reader, exception=ex)
-
-    def _read_batches(self, context) -> Iterator[StreamBatch]:
-        batch_entries = []
-        batch_number = 0
-
-        def batch_maker(last=False):
-            nonlocal batch_number, batch_entries
-            batch_number += 1
-            ret = StreamBatch(
-                entries=batch_entries,
-                seq=batch_number,
-                context=copy.deepcopy(context),
-                last=last,
-            )
-            batch_entries = []
-            return ret
-
-        for entry in self._read_entries():
-            if len(batch_entries) == self._batch_size:
-                batch = batch_maker()
-                if self._reader_callback:
-                    self._reader_callback(batch)
-                yield batch
-                batch_entries = []
-            batch_entries.append(entry)
-        batch = batch_maker(last=True)
-        if self._reader_callback:
-            self._reader_callback(batch)
-        yield batch
-
-    def process(self, context=None, identity=system_identity):
-        context = context or {}
-        chain = self.build_chain(identity)
-        for batch in self._read_batches(context):
-            chain.process(batch, self._callback)
-
-    @abc.abstractmethod
-    def build_chain(self, identity) -> DataStreamChain:
-        pass
-
-    def _reader_error(self, reader, exception):
-        self._callback.reader_error(reader, exception=exception)

oarepo_runtime/datastreams/errors.py

@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2021 CERN.
-#
-# Invenio-Vocabularies is free software; you can redistribute it and/or
-# modify it under the terms of the MIT License; see LICENSE file for more
-# details.
-
-"""Datastream errors."""
-from typing import Union
-
-from .json import JSONObject
-
-
-class DataStreamError(Exception):
-    def __init__(
-        self,
-        message,
-        code=None,
-        location=None,
-        detail: Union[JSONObject, None] = None,
-    ):
-        """
-        @param message: a string message (overview)
-        @param code: a machine processable code
-        @param location: location inside the json, where the error was detected. Using dot notation,
-               arrays are indexed from 0, for example: `metadata.titles.0.language`
-        @param detail: a json-serializable object (dictionary) with details
-        """
-        super().__init__(message)
-        assert detail is None or isinstance(detail, dict)
-        self.detail = detail
-        self.message = message
-        self.code = code
-        self.location = location
-
-
-class ReaderError(DataStreamError):
-    """Transformer application exception."""
-
-
-class TransformerError(DataStreamError):
-    """Transformer application exception."""
-
-
-class WriterError(DataStreamError):
-    """Transformer application exception."""
-
-
-class DataStreamCatalogueError(Exception):
-    def __init__(self, message, entry=None, stream_name=None) -> None:
-        super().__init__(message)
-        self.entry = entry
-        self.stream_name = stream_name

oarepo_runtime/datastreams/ext.py

@@ -1,41 +0,0 @@
-import functools
-
-from invenio_base.utils import obj_or_import_string
-
-from oarepo_runtime.datastreams.datastreams import Signature
-
-
-class OARepoDataStreamsExt:
-    def __init__(self, app):
-        self.app = app
-
-    def get_reader(self, reader, identity, **kwargs):
-        return self._get_instance("DATASTREAMS_READERS", identity, kwargs, reader)
-
-    def get_writer(self, writer, identity, **kwargs):
-        return self._get_instance("DATASTREAMS_WRITERS", identity, kwargs, writer)
-
-    def get_transformer(self, transformer, identity, **kwargs):
-        return self._get_instance(
-            "DATASTREAMS_TRANSFORMERS", identity, kwargs, transformer
-        )
-
-    def _get_instance(self, config_name, identity, kwargs, inst):
-        if isinstance(inst, Signature):
-            config_classes = self._get_classes_from_config(config_name)
-            if inst.name not in config_classes:
-                raise KeyError(f"'{inst.name}' not found in config {config_name}")
-            reader_class = config_classes[inst.name]
-            all_kwargs = {**(inst.kwargs or {}), **kwargs}
-            if "identity" not in all_kwargs:
-                all_kwargs["identity"] = identity
-            return reader_class(**all_kwargs)
-        else:
-            return inst
-
-    @functools.lru_cache(maxsize=5)
-    def _get_classes_from_config(self, config_name):
-        return {
-            class_key: obj_or_import_string(class_name)
-            for class_key, class_name in self.app.config[config_name].items()
-        }

oarepo_runtime/datastreams/fixtures.py

@@ -1,265 +0,0 @@
-import logging
-import re
-from pathlib import Path
-
-import pkg_resources
-import yaml
-from celery import shared_task
-from flask import current_app
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.datastreams import (
-    DataStreamCatalogue,
-    StreamBatch,
-    SynchronousDataStream,
-)
-from oarepo_runtime.datastreams.types import StatsKeepingDataStreamCallback
-
-log = logging.getLogger("fixtures")
-
-
-class FixturesCallback(StatsKeepingDataStreamCallback):
-    def fixture_started(self, fixture_name):
-        pass
-
-    def fixture_finished(self, fixture_name):
-        pass
-
-
-def load_fixtures(
-    fixture_dir_or_catalogue=None,
-    include=None,
-    exclude=None,
-    system_fixtures=True,
-    callback: FixturesCallback = None,
-    batch_size=100,
-    datastreams_impl=SynchronousDataStream,
-    identity=system_identity,
-):
-    """
-    Loads fixtures. If fixture dir is set, fixtures are loaded from that directory first.
-    The directory must contain a catalogue.yaml file containing datastreams to load the
-    fixtures. The format of the catalogue is described in the 'catalogue.py' file.
-
-    Then fixture loading continues with fixtures defined in `oarepo.fixtures` entrypoint.
-    The entry points are sorted and those with the greatest `name` are processed first -
-    so the recommendation is to call the entry points 0000-something, where 0000 is a 4-digit
-    number. oarepo entry points always have this number set to 1000.
-
-    If a datastream is loaded from one fixture, it will not be loaded again from another fixture.
-    If you want to override the default fixtures, just register your own with a key bigger than 1000.
-    """
-    include = [re.compile(x) for x in (include or [])]
-    exclude = [re.compile(x) for x in (exclude or [])]
-    fixtures = set()
-
-    if fixture_dir_or_catalogue:
-        if Path(fixture_dir_or_catalogue).is_dir():
-            fixture_catalogue = Path(fixture_dir_or_catalogue) / "catalogue.yaml"
-        else:
-            fixture_catalogue = Path(fixture_dir_or_catalogue)
-
-        catalogue = DataStreamCatalogue(fixture_catalogue)
-        _load_fixtures_from_catalogue(
-            catalogue,
-            fixtures,
-            include,
-            exclude,
-            callback,
-            batch_size=batch_size,
-            datastreams_impl=datastreams_impl,
-            identity=identity,
-        )
-
-    if system_fixtures:
-
-        def get_priority(name):
-            match = re.match(r"(\d+)-", name)
-            if match:
-                return -int(match.group(1))
-            return 0
-
-        entry_points = list(
-            (get_priority(r.name), r.name, r)
-            for r in pkg_resources.iter_entry_points("oarepo.fixtures")
-        )
-        entry_points.sort(key=lambda x: x[:2])
-        for r in entry_points:
-            pkg = r[2].load()
-            pkg_fixture_dir = Path(pkg.__file__)
-            if pkg_fixture_dir.is_file():
-                pkg_fixture_dir = pkg_fixture_dir.parent
-            catalogue = DataStreamCatalogue(pkg_fixture_dir / "catalogue.yaml")
-            _load_fixtures_from_catalogue(
-                catalogue,
-                fixtures,
-                include,
-                exclude,
-                callback,
-                batch_size=batch_size,
-                datastreams_impl=datastreams_impl,
-                identity=identity,
-            )
-
-
-def _load_fixtures_from_catalogue(
-    catalogue,
-    fixtures,
-    include,
-    exclude,
-    callback,
-    batch_size,
-    datastreams_impl,
-    identity=system_identity,
-):
-    for catalogue_datastream in catalogue.get_datastreams():
-        if catalogue_datastream.stream_name in fixtures:
-            continue
-        if include and not any(
-            x.match(catalogue_datastream.stream_name) for x in include
-        ):
-            continue
-        if any(x.match(catalogue_datastream.stream_name) for x in exclude):
-            continue
-
-        fixtures.add(catalogue_datastream.stream_name)
-
-        if hasattr(callback, "fixture_started"):
-            callback.fixture_started(catalogue_datastream.stream_name)
-        datastream = datastreams_impl(
-            readers=catalogue_datastream.readers,
-            writers=catalogue_datastream.writers,
-            transformers=catalogue_datastream.transformers,
-            callback=callback,
-            batch_size=batch_size,
-        )
-        datastream.process(identity=identity)
-        if hasattr(callback, "fixture_finished"):
-            callback.fixture_finished(catalogue_datastream.stream_name)
-
-
-def dump_fixtures(
-    fixture_dir,
-    include=None,
-    exclude=None,
-    use_files=False,
-    callback: FixturesCallback = None,
-    datastream_impl=SynchronousDataStream,
-    batch_size=1,
-):
-    include = [re.compile(x) for x in (include or [])]
-    exclude = [
-        re.compile(x)
-        for x in (exclude or current_app.config.get("DATASTREAMS_EXCLUDES", []))
-    ]
-    fixture_dir = Path(fixture_dir)
-    if not fixture_dir.exists():
-        fixture_dir.mkdir(parents=True)
-    catalogue_path = fixture_dir / "catalogue.yaml"
-    catalogue_data = {}
-
-    for service_id in current_service_registry._services:
-        config_generator = (
-            current_app.config.get(f"DATASTREAMS_CONFIG_GENERATOR_{service_id.upper()}")
-            or current_app.config["DATASTREAMS_CONFIG_GENERATOR"]
-        )
-        service = current_service_registry.get(service_id)
-        if not hasattr(service, "scan"):
-            continue
-        for fixture_name, fixture_read_config, fixture_write_config in config_generator(
-            service_id, use_files=use_files
-        ):
-            if include and not any(x.match(fixture_name) for x in include):
-                continue
-            if any(x.match(fixture_name) for x in exclude):
-                continue
-
-            catalogue_data[fixture_name] = fixture_read_config
-
-            catalogue = DataStreamCatalogue(
-                catalogue_path, {fixture_name: fixture_write_config}
-            )
-
-            for stream_name in catalogue:
-                catalogue_datastream = catalogue.get_datastream(stream_name)
-                if hasattr(callback, "fixture_started"):
-                    callback.fixture_started(stream_name)
-                datastream = datastream_impl(
-                    readers=catalogue_datastream.readers,
-                    writers=catalogue_datastream.writers,
-                    transformers=catalogue_datastream.transformers,
-                    callback=callback,
-                    batch_size=batch_size,
-                )
-                datastream.process()
-                if hasattr(callback, "fixture_finished"):
-                    callback.fixture_finished(stream_name)
-
-    with open(catalogue_path, "w") as f:
-        yaml.dump(catalogue_data, f)
-
-
-def default_config_generator(service_id, use_files=False):
-    writers = [
-        {"writer": "yaml", "target": f"{service_id}.yaml"},
-    ]
-    if use_files:
-        writers.append(
-            {"writer": "attachments_file", "target": "files"},
-        )
-
-    yield service_id, [
-        # load
-        {"writer": "service", "service": service_id},
-        {"writer": "attachments_service", "service": service_id},
-        {"source": f"{service_id}.yaml"},
-    ], [
-        # dump
-        {"reader": "service", "service": service_id, "load_files": use_files},
-        *writers,
-    ]
-
-
-@shared_task
-def fixtures_asynchronous_callback(*args, callback, **kwargs):
-    try:
-        if "batch" in kwargs:
-            batch = StreamBatch.from_json(kwargs["batch"])
-            log.info(
-                "Fixtures progress: %s in batch.seq=%s, batch.last=%s",
-                callback,
-                batch.seq,
-                batch.last,
-            )
-        else:
-            batch = None
-            log.info("Fixtures progress: %s", callback)
-
-        if "error" in callback:
-            log.error(
-                "Error in loading fixtures: %s\n%s\n%s",
-                callback,
-                "\n".join(args),
-                "\n".join(f"{kwarg}: {value}" for kwarg, value in kwargs.items()),
-            )
-
-        if batch:
-            if batch.errors:
-                log.error(
-                    "Batch errors: batch %s:\n%s",
-                    batch.seq,
-                    "\n".join(str(x) for x in batch.errors),
-                )
-
-            for entry in batch.entries:
-                if entry.errors:
-                    log.error(
-                        "Errors in entry %s of batch %s:\npayload %s\n",
-                        entry.seq,
-                        batch.seq,
-                        entry.entry,
-                        "\n".join(str(x) for x in entry.errors),
-                    )
-    except Exception:
-        print(f"Error in fixtures callback: {callback=}, {args=}, {kwargs=}")

oarepo_runtime/datastreams/json.py

@@ -1,4 +0,0 @@
-from typing import Union
-
-JSON = Union[str, int, float, bool, None, dict[str, "JSON"], list["JSON"]]
-JSONObject = dict[str, "JSON"]

oarepo_runtime/datastreams/readers/__init__.py

@@ -1,39 +0,0 @@
-import contextlib
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import Iterator, Union
-
-from ..types import StreamEntry
-
-
-class BaseReader(ABC):
-    """Base reader."""
-
-    base_path: Union[Path, None]
-
-    def __init__(self, *, source=None, base_path=None, **kwargs):
-        """Constructor.
-        :param source: Data source (e.g. filepath, stream, ...)
-        """
-        if not source or hasattr(source, "read") or not base_path:
-            self.source = source
-        else:
-            self.source = Path(base_path).joinpath(source)
-        if base_path:
-            self.base_path = Path(base_path)
-        elif isinstance(source, (str, Path)):
-            self.base_path = Path(source).parent
-        else:
-            self.base_path = None
-
-    @abstractmethod
-    def __iter__(self) -> Iterator[StreamEntry]:
-        """Yields data objects."""
-
-    @contextlib.contextmanager
-    def _open(self, mode="r"):
-        if hasattr(self.source, "read"):
-            yield self.source
-        else:
-            with open(self.source, mode) as f:
-                yield f

oarepo_runtime/datastreams/readers/attachments.py

@@ -1,51 +0,0 @@
-from base64 import b64encode
-from pathlib import Path
-
-import yaml
-
-from oarepo_runtime.datastreams import BaseReader, StreamEntry
-from oarepo_runtime.datastreams.types import StreamEntryFile
-from oarepo_runtime.datastreams.writers.attachments_file import format_serial
-
-
-class AttachmentsReaderMixin(BaseReader):
-    def __init__(self, *, source=None, base_path=None, **kwargs):
-        super().__init__(source=source, base_path=base_path, **kwargs)
-        self.has_files = self.base_path and (self.base_path / "files").is_dir()
-
-    def __iter__(self):
-        """Iterate over records."""
-        se: StreamEntry
-        for idx, se in enumerate(self.iter_entries()):
-            if self.has_files:
-                file_path = (
-                    self.base_path.joinpath("files", format_serial(idx + 1)) / "data"
-                )
-                if file_path.exists():
-                    file_metadata = self.load_file_metadata(file_path)
-                    for md in file_metadata:
-                        se.files.append(
-                            StreamEntryFile(
-                                metadata=md,
-                                content_url="data:"
-                                + b64encode(
-                                    (file_path / md["key"]).read_bytes()
-                                ).decode("ascii"),
-                            )
-                        )
-            yield se
-
-    def load_file_metadata(self, file_path: Path):
-        md = "metadata.yaml"
-        while True:
-            tested_md = "meta_" + md
-            # meta_[A]metadata.yaml does not exist, so [A]metadata.yaml is the metadata file,
-            # where A is (meta_)*
-            if not (file_path / tested_md).exists():
-                with open(file_path / md) as f:
-                    return list(yaml.safe_load_all(f))
-            md = tested_md
-
-    def iter_entries(self):
-        "Return an iterator of entries"
-        return []