oarepo-runtime 1.10.2__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/datastreams/fixtures.py

@@ -1,265 +0,0 @@
-import logging
-import re
-from pathlib import Path
-
-import pkg_resources
-import yaml
-from celery import shared_task
-from flask import current_app
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.datastreams import (
-    DataStreamCatalogue,
-    StreamBatch,
-    SynchronousDataStream,
-)
-from oarepo_runtime.datastreams.types import StatsKeepingDataStreamCallback
-
-log = logging.getLogger("fixtures")
-
-
-class FixturesCallback(StatsKeepingDataStreamCallback):
-    def fixture_started(self, fixture_name):
-        pass
-
-    def fixture_finished(self, fixture_name):
-        pass
-
-
-def load_fixtures(
-    fixture_dir_or_catalogue=None,
-    include=None,
-    exclude=None,
-    system_fixtures=True,
-    callback: FixturesCallback = None,
-    batch_size=100,
-    datastreams_impl=SynchronousDataStream,
-    identity=system_identity,
-):
-    """
-    Loads fixtures. If fixture dir is set, fixtures are loaded from that directory first.
-    The directory must contain a catalogue.yaml file containing datastreams to load the
-    fixtures. The format of the catalogue is described in the 'catalogue.py' file.
-
-    Then fixture loading continues with fixtures defined in `oarepo.fixtures` entrypoint.
-    The entry points are sorted and those with the greatest `name` are processed first -
-    so the recommendation is to call the entry points 0000-something, where 0000 is a 4-digit
-    number. oarepo entry points always have this number set to 1000.
-
-    If a datastream is loaded from one fixture, it will not be loaded again from another fixture.
-    If you want to override the default fixtures, just register your own with a key bigger than 1000.
-    """
-    include = [re.compile(x) for x in (include or [])]
-    exclude = [re.compile(x) for x in (exclude or [])]
-    fixtures = set()
-
-    if fixture_dir_or_catalogue:
-        if Path(fixture_dir_or_catalogue).is_dir():
-            fixture_catalogue = Path(fixture_dir_or_catalogue) / "catalogue.yaml"
-        else:
-            fixture_catalogue = Path(fixture_dir_or_catalogue)
-
-        catalogue = DataStreamCatalogue(fixture_catalogue)
-        _load_fixtures_from_catalogue(
-            catalogue,
-            fixtures,
-            include,
-            exclude,
-            callback,
-            batch_size=batch_size,
-            datastreams_impl=datastreams_impl,
-            identity=identity,
-        )
-
-    if system_fixtures:
-
-        def get_priority(name):
-            match = re.match(r"(\d+)-", name)
-            if match:
-                return -int(match.group(1))
-            return 0
-
-        entry_points = list(
-            (get_priority(r.name), r.name, r)
-            for r in pkg_resources.iter_entry_points("oarepo.fixtures")
-        )
-        entry_points.sort(key=lambda x: x[:2])
-        for r in entry_points:
-            pkg = r[2].load()
-            pkg_fixture_dir = Path(pkg.__file__)
-            if pkg_fixture_dir.is_file():
-                pkg_fixture_dir = pkg_fixture_dir.parent
-            catalogue = DataStreamCatalogue(pkg_fixture_dir / "catalogue.yaml")
-            _load_fixtures_from_catalogue(
-                catalogue,
-                fixtures,
-                include,
-                exclude,
-                callback,
-                batch_size=batch_size,
-                datastreams_impl=datastreams_impl,
-                identity=identity,
-            )
-
-
-def _load_fixtures_from_catalogue(
-    catalogue,
-    fixtures,
-    include,
-    exclude,
-    callback,
-    batch_size,
-    datastreams_impl,
-    identity=system_identity,
-):
-    for catalogue_datastream in catalogue.get_datastreams():
-        if catalogue_datastream.stream_name in fixtures:
-            continue
-        if include and not any(
-            x.match(catalogue_datastream.stream_name) for x in include
-        ):
-            continue
-        if any(x.match(catalogue_datastream.stream_name) for x in exclude):
-            continue
-
-        fixtures.add(catalogue_datastream.stream_name)
-
-        if hasattr(callback, "fixture_started"):
-            callback.fixture_started(catalogue_datastream.stream_name)
-        datastream = datastreams_impl(
-            readers=catalogue_datastream.readers,
-            writers=catalogue_datastream.writers,
-            transformers=catalogue_datastream.transformers,
-            callback=callback,
-            batch_size=batch_size,
-        )
-        datastream.process(identity=identity)
-        if hasattr(callback, "fixture_finished"):
-            callback.fixture_finished(catalogue_datastream.stream_name)
-
-
-def dump_fixtures(
-    fixture_dir,
-    include=None,
-    exclude=None,
-    use_files=False,
-    callback: FixturesCallback = None,
-    datastream_impl=SynchronousDataStream,
-    batch_size=1,
-):
-    include = [re.compile(x) for x in (include or [])]
-    exclude = [
-        re.compile(x)
-        for x in (exclude or current_app.config.get("DATASTREAMS_EXCLUDES", []))
-    ]
-    fixture_dir = Path(fixture_dir)
-    if not fixture_dir.exists():
-        fixture_dir.mkdir(parents=True)
-    catalogue_path = fixture_dir / "catalogue.yaml"
-    catalogue_data = {}
-
-    for service_id in current_service_registry._services:
-        config_generator = (
-            current_app.config.get(f"DATASTREAMS_CONFIG_GENERATOR_{service_id.upper()}")
-            or current_app.config["DATASTREAMS_CONFIG_GENERATOR"]
-        )
-        service = current_service_registry.get(service_id)
-        if not hasattr(service, "scan"):
-            continue
-        for fixture_name, fixture_read_config, fixture_write_config in config_generator(
-            service_id, use_files=use_files
-        ):
-            if include and not any(x.match(fixture_name) for x in include):
-                continue
-            if any(x.match(fixture_name) for x in exclude):
-                continue
-
-            catalogue_data[fixture_name] = fixture_read_config
-
-            catalogue = DataStreamCatalogue(
-                catalogue_path, {fixture_name: fixture_write_config}
-            )
-
-            for stream_name in catalogue:
-                catalogue_datastream = catalogue.get_datastream(stream_name)
-                if hasattr(callback, "fixture_started"):
-                    callback.fixture_started(stream_name)
-                datastream = datastream_impl(
-                    readers=catalogue_datastream.readers,
-                    writers=catalogue_datastream.writers,
-                    transformers=catalogue_datastream.transformers,
-                    callback=callback,
-                    batch_size=batch_size,
-                )
-                datastream.process()
-                if hasattr(callback, "fixture_finished"):
-                    callback.fixture_finished(stream_name)
-
-    with open(catalogue_path, "w") as f:
-        yaml.dump(catalogue_data, f)
-
-
-def default_config_generator(service_id, use_files=False):
-    writers = [
-        {"writer": "yaml", "target": f"{service_id}.yaml"},
-    ]
-    if use_files:
-        writers.append(
-            {"writer": "attachments_file", "target": "files"},
-        )
-
-    yield service_id, [
-        # load
-        {"writer": "service", "service": service_id},
-        {"writer": "attachments_service", "service": service_id},
-        {"source": f"{service_id}.yaml"},
-    ], [
-        # dump
-        {"reader": "service", "service": service_id, "load_files": use_files},
-        *writers,
-    ]
-
-
-@shared_task
-def fixtures_asynchronous_callback(*args, callback, **kwargs):
-    try:
-        if "batch" in kwargs:
-            batch = StreamBatch.from_json(kwargs["batch"])
-            log.info(
-                "Fixtures progress: %s in batch.seq=%s, batch.last=%s",
-                callback,
-                batch.seq,
-                batch.last,
-            )
-        else:
-            batch = None
-            log.info("Fixtures progress: %s", callback)
-
-        if "error" in callback:
-            log.error(
-                "Error in loading fixtures: %s\n%s\n%s",
-                callback,
-                "\n".join(args),
-                "\n".join(f"{kwarg}: {value}" for kwarg, value in kwargs.items()),
-            )
-
-        if batch:
-            if batch.errors:
-                log.error(
-                    "Batch errors: batch %s:\n%s",
-                    batch.seq,
-                    "\n".join(str(x) for x in batch.errors),
-                )
-
-            for entry in batch.entries:
-                if entry.errors:
-                    log.error(
-                        "Errors in entry %s of batch %s:\npayload %s\n",
-                        entry.seq,
-                        batch.seq,
-                        entry.entry,
-                        "\n".join(str(x) for x in entry.errors),
-                    )
-    except Exception:
-        print(f"Error in fixtures callback: {callback=}, {args=}, {kwargs=}")

oarepo_runtime/datastreams/json.py

@@ -1,4 +0,0 @@
-from typing import Union
-
-JSON = Union[str, int, float, bool, None, dict[str, "JSON"], list["JSON"]]
-JSONObject = dict[str, "JSON"]

oarepo_runtime/datastreams/readers/__init__.py

@@ -1,39 +0,0 @@
-import contextlib
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import Iterator, Union
-
-from ..types import StreamEntry
-
-
-class BaseReader(ABC):
-    """Base reader."""
-
-    base_path: Union[Path, None]
-
-    def __init__(self, *, source=None, base_path=None, **kwargs):
-        """Constructor.
-        :param source: Data source (e.g. filepath, stream, ...)
-        """
-        if not source or hasattr(source, "read") or not base_path:
-            self.source = source
-        else:
-            self.source = Path(base_path).joinpath(source)
-        if base_path:
-            self.base_path = Path(base_path)
-        elif isinstance(source, (str, Path)):
-            self.base_path = Path(source).parent
-        else:
-            self.base_path = None
-
-    @abstractmethod
-    def __iter__(self) -> Iterator[StreamEntry]:
-        """Yields data objects."""
-
-    @contextlib.contextmanager
-    def _open(self, mode="r"):
-        if hasattr(self.source, "read"):
-            yield self.source
-        else:
-            with open(self.source, mode) as f:
-                yield f

oarepo_runtime/datastreams/readers/attachments.py

@@ -1,51 +0,0 @@
-from base64 import b64encode
-from pathlib import Path
-
-import yaml
-
-from oarepo_runtime.datastreams import BaseReader, StreamEntry
-from oarepo_runtime.datastreams.types import StreamEntryFile
-from oarepo_runtime.datastreams.writers.attachments_file import format_serial
-
-
-class AttachmentsReaderMixin(BaseReader):
-    def __init__(self, *, source=None, base_path=None, **kwargs):
-        super().__init__(source=source, base_path=base_path, **kwargs)
-        self.has_files = self.base_path and (self.base_path / "files").is_dir()
-
-    def __iter__(self):
-        """Iterate over records."""
-        se: StreamEntry
-        for idx, se in enumerate(self.iter_entries()):
-            if self.has_files:
-                file_path = (
-                    self.base_path.joinpath("files", format_serial(idx + 1)) / "data"
-                )
-                if file_path.exists():
-                    file_metadata = self.load_file_metadata(file_path)
-                    for md in file_metadata:
-                        se.files.append(
-                            StreamEntryFile(
-                                metadata=md,
-                                content_url="data:"
-                                + b64encode(
-                                    (file_path / md["key"]).read_bytes()
-                                ).decode("ascii"),
-                            )
-                        )
-            yield se
-
-    def load_file_metadata(self, file_path: Path):
-        md = "metadata.yaml"
-        while True:
-            tested_md = "meta_" + md
-            # meta_[A]metadata.yaml does not exist, so [A]metadata.yaml is the metadata file,
-            # where A is (meta_)*
-            if not (file_path / tested_md).exists():
-                with open(file_path / md) as f:
-                    return list(yaml.safe_load_all(f))
-            md = tested_md
-
-    def iter_entries(self):
-        "Return an iterator of entries"
-        return []

oarepo_runtime/datastreams/readers/excel.py

@@ -1,123 +0,0 @@
-import re
-from traceback import format_exc
-from typing import Iterator
-
-import openpyxl
-
-from ..errors import ReaderError
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class ExcelReader(AttachmentsReaderMixin, BaseReader):
-    def iter_entries(self) -> Iterator[StreamEntry]:
-        with self._open("rb") as f:
-            try:
-                wb_obj = openpyxl.load_workbook(f)
-                sheet_obj = wb_obj.active
-            except Exception as err:
-                raise ReaderError(
-                    f"Cannot decode excel file {self._data_file.name}: {str(err)}",
-                    code="EXCEL_DECODE_ERROR",
-                    detail={
-                        "message": str(err),
-                        "exception": type(err).__name__,
-                        "stack": format_exc(limit=10),
-                    },
-                )
-
-            header, data = self.get_excel_data(sheet_obj)
-            for row in data:
-                yield StreamEntry(row)
-
-    def get_excel_data(self, sheet_obj):
-        """
-        returns an iterator (header, data)
-        """
-        header = []
-        data = []
-        it = sheet_obj.iter_rows()
-
-        try:
-            row = next_row(it)
-            while empty(row):
-                row = next_row(it)
-            while not empty(row):
-                header.append(row)
-                row = next_row(it)
-            while empty(row):
-                row = next_row(it)
-            while True:
-                if not empty(row):
-                    data.append(row)
-                row = next_row(it)
-        except StopIteration:
-            pass
-        if not data:
-            return [], self.to_dict(header)
-        else:
-            return self.to_dict(header), self.to_dict(data)
-
-    def to_dict(self, dta):
-        def is_array(val):
-            try:
-                int(val)
-                return True
-            except:
-                return False
-
-        def set_single(container, key, val):
-            try:
-                key = int(key)
-                while key >= len(container):
-                    container.append(None)
-                container[key] = val
-            except (TypeError, ValueError):
-                container[key] = val
-
-        def iterset(k, v, container):
-            while True:
-                current_key = k[0]
-                next_key = k[1] if len(k) > 1 else None
-                if not next_key:
-                    set_single(container, current_key, v)
-                    return
-                if isinstance(container, list):
-                    container.append({} if not is_array(next_key) else [])
-                    container = container[-1]
-                else:
-                    container = container.setdefault(
-                        current_key, {} if not is_array(next_key) else []
-                    )
-                k = k[1:]
-
-        def to_dict_item(header, item):
-            ret = RowDict()
-            for k, v in zip(header, item):
-                if not k:
-                    continue
-                v = v if v is not None else ""
-                v = str(v).strip()
-                if v:
-                    iterset(k, v, ret)
-            return ret
-
-        keys = [re.split("[_.]", x) if x else None for x in dta[0]]
-        return [to_dict_item(keys, d) for d in dta[1:]]
-
-
-def next_row(it):
-    return [x.value for x in next(it)]
-
-
-def empty(r):
-    for val in r:
-        if val:
-            return False
-    return True
-
-
-class RowDict(dict):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._header = {}

oarepo_runtime/datastreams/readers/json.py

@@ -1,27 +0,0 @@
-import json
-from typing import Iterator
-
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class JSONReader(AttachmentsReaderMixin, BaseReader):
-    """JSON Lines data iterator that loads records from JSON Lines files."""
-
-    def iter_entries(self) -> Iterator[StreamEntry]:
-        """Iterate over records."""
-        with self._open() as fp:
-            data = json.load(fp)
-            assert isinstance(data, list)
-            for d in data:
-                yield StreamEntry(d)
-
-
-class JSONLinesReader(BaseReader):
-    """JSON Lines data iterator that loads records from JSON Lines files."""
-
-    def __iter__(self) -> Iterator[StreamEntry]:
-        """Iterate over records."""
-        with self._open() as fp:
-            for line in fp:
-                yield StreamEntry(json.loads(line))

oarepo_runtime/datastreams/readers/service.py

@@ -1,54 +0,0 @@
-from base64 import b64encode
-from typing import List
-
-from invenio_access.permissions import system_identity
-from invenio_records_resources.proxies import current_service_registry
-
-from ..types import StreamEntryFile
-from ..utils import get_file_service_for_record_class
-from . import BaseReader, StreamEntry
-
-
-class ServiceReader(BaseReader):
-    """Writes the entries to a repository instance using a Service object."""
-
-    def __init__(self, *, service=None, identity=None, load_files=False, **kwargs):
-        """Constructor.
-        :param service_or_name: a service instance or a key of the
-                                service registry.
-        :param identity: access identity.
-        :param update: if True it will update records if they exist.
-        """
-        super().__init__(**kwargs)
-
-        if isinstance(service, str):
-            service = current_service_registry.get(service)
-
-        self._service = service
-        self._identity = identity or system_identity
-        self._file_service = None
-        self._record_cls = getattr(self._service.config, "record_cls", None)
-
-        if self._record_cls and load_files:
-            # try to get file service
-            self._file_service = get_file_service_for_record_class(self._record_cls)
-
-    def __iter__(self):
-        for idx, entry in enumerate(self._service.scan(self._identity)):
-            files: List[StreamEntryFile] = []
-            if self._file_service:
-                for f in self._file_service.list_files(
-                    self._identity, entry["id"]
-                ).entries:
-                    file_item = self._file_service.get_file_content(
-                        self._identity, entry["id"], f["key"]
-                    )
-                    with file_item.open_stream("rb") as ff:
-                        base64_content = b64encode(ff.read()).decode("ascii")
-                        files.append(
-                            StreamEntryFile(
-                                metadata=f, content_url=f"data:{base64_content}"
-                            )
-                        )
-
-            yield StreamEntry(entry, files=files)

oarepo_runtime/datastreams/readers/yaml.py

@@ -1,14 +0,0 @@
-import yaml
-
-from . import BaseReader, StreamEntry
-from .attachments import AttachmentsReaderMixin
-
-
-class YamlReader(AttachmentsReaderMixin, BaseReader):
-    """YAML data iterator that loads records from YAML files."""
-
-    def iter_entries(self):
-        """Iterate over records."""
-        with self._open() as fp:
-            for entry in yaml.safe_load_all(fp):
-                yield StreamEntry(entry)

oarepo_runtime/datastreams/semi_asynchronous.py

@@ -1,91 +0,0 @@
-#
-# This package was taken from Invenio vocabularies and modified to be more universal
-#
-import logging
-from typing import List
-
-from celery import shared_task
-from celery.canvas import Signature as CelerySignature
-
-from .asynchronous import (
-    AsynchronousDataStream,
-    AsynchronousDataStreamChain,
-    deserialize_identity,
-    serialize_identity,
-)
-from .datastreams import DataStreamChain, Signature
-from .transformers import BaseTransformer
-from .types import JSONObject, StreamBatch, StreamEntryError
-
-log = logging.getLogger("datastreams")
-
-
-class SemiAsynchronousDataStreamChain(AsynchronousDataStreamChain):
-    def _prepare_chain(self, callback: CelerySignature):
-        serialized_identity = serialize_identity(self._identity)
-        return run_semi_asynchronous_datastream_processor.s(
-            transformers=[tr.json for tr in self._transformers],
-            writers=[wr.json for wr in self._writers],
-            identity=serialized_identity,
-            callback=callback,
-        )
-
-
-class SemiAsynchronousDataStream(AsynchronousDataStream):
-    """Data stream."""
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return SemiAsynchronousDataStreamChain(
-            transformers=self._transformers,
-            writers=self._writers,
-            on_background=self._on_background,
-            identity=identity,
-        )
-
-
-@shared_task
-def run_semi_asynchronous_datastream_processor(
-    batch: JSONObject,
-    *,
-    transformers: List[JSONObject],
-    writers: List[JSONObject],
-    identity: JSONObject,
-    callback: CelerySignature,
-):
-    """Run datastream processor."""
-
-    callback.apply(kwargs={"callback": "batch_started", "batch": batch})
-
-    batch = StreamBatch.from_json(batch)
-    identity = deserialize_identity(identity)
-
-    for signature in (transformers or []) + (writers or []):
-        signature = Signature.from_json(signature)
-        try:
-            processor = signature.resolve(identity=identity)
-            if isinstance(processor, BaseTransformer):
-                batch = processor.apply(batch) or batch
-            else:
-                batch = processor.write(batch) or batch
-        except Exception as ex:
-            if log.getEffectiveLevel():
-                log.error(
-                    "Unexpected error in %s: %s",
-                    repr(signature),
-                    repr(batch),
-                )
-            err = StreamEntryError.from_exception(ex)
-            batch.errors.append(err)
-            callback.apply(
-                (),
-                {
-                    "batch": batch.json,
-                    "identity": serialize_identity(identity),
-                    "callback": f"{signature.kind.value}_error",
-                    "exception": err.json,
-                },
-            )
-
-    callback.apply(kwargs={"callback": "batch_finished", "batch": batch.json})
-
-    return None  # do not return anything to avoid redis pollution