oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime/cli/permissions/search.py

@@ -1,121 +0,0 @@
-import json
-import sys
-
-import click
-import yaml
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.info.permissions.debug import add_debugging, merge_communities
-
-from .base import get_user_and_identity, permissions
-
-
-@permissions.command(name="search")
-@click.argument("user_id_or_email")
-@click.argument("service_name")
-@click.option("--explain/--no-explain", default=False)
-@click.option("--user/--published", "user_call", default=False)
-@click.option("--full-query/--query-filters", default=False)
-@click.option("--merge-communities", "do_merge_communities", is_flag=True)
-@click.option("--json/--yaml", "as_json", default=False)
-def search_permissions(
-    user_id_or_email,
-    service_name,
-    explain,
-    user_call,
-    full_query,
-    do_merge_communities,
-    as_json,
-):
-    """Get search parameters for a given service."""
-    try:
-        service = current_service_registry.get(service_name)
-    except KeyError:
-        raise click.UsageError(
-            f"Service {service_name} not found in {current_service_registry._services.keys()}"
-        )
-    user, identity = get_user_and_identity(user_id_or_email)
-
-    permission_policy = service.config.permission_policy_cls
-
-    add_debugging(print_search=explain, print_needs=False, print_excludes=False)
-
-    if full_query:
-        previous_search = service._search
-
-        class NoExecute:
-            def __init__(self, query):
-                self.query = query
-
-            def execute(self):
-                return self.query
-
-        def _patched_search(*args, **kwargs):
-            ret = previous_search(*args, **kwargs)
-            return NoExecute(ret)
-
-        def _patched_result_list(self, identity, results, params, **kwargs):
-            return results
-
-        service._search = _patched_search
-        service.result_list = _patched_result_list
-
-        if user_call:
-            ret = service.search_drafts(identity)
-        else:
-            ret = service.search(identity)
-        ret = ret.to_dict()
-        if do_merge_communities:
-            ret = merge_communities(ret)
-        ret = {
-            "query": ret["query"],
-        }
-        dump_dict(ret, as_json)
-    else:
-
-        over = {}
-        if explain:
-            over["debug_identity"] = identity
-            print("## Explaining search:")
-
-        if user_call:
-            p = permission_policy("read_draft", identity=identity, **over)
-        else:
-            p = permission_policy("read_deleted", identity=identity, **over)
-        query_filters = p.query_filters
-
-        print()
-        print("## Query filters:")
-        for qf in query_filters:
-            dict_qf = qf.to_dict()
-            if explain:
-                dict_qf = merge_communities(dict_qf)
-            dump_dict(dict_qf, as_json)
-            print(json.dumps(dict_qf, indent=2))
-
-
-def merge_name(d):
-    if isinstance(d, list):
-        return [merge_name(x) for x in d]
-    if isinstance(d, dict):
-        ret = {}
-        for k, v in d.items():
-            v = merge_name(v)
-            if isinstance(v, dict) and "_name" in v:
-                _name = v.pop("_name")
-                _name = _name.split("@")[0].strip()
-                k = f"{k}[{_name}]"
-            ret[k] = v
-        return ret
-    return d
-
-
-def dump_dict(d, as_json=False):
-    if as_json:
-        print(json.dumps(d, indent=2))
-    else:
-        yaml.safe_dump(
-            merge_name(json.loads(json.dumps(d))),
-            sys.stdout,
-            default_flow_style=False,
-        )

oarepo_runtime/cli/validate.py

@@ -1,150 +0,0 @@
-import sys
-import traceback
-
-import click
-import yaml
-from flask.cli import with_appcontext
-from invenio_db import db
-from invenio_records import Record
-from invenio_records_resources.proxies import current_service_registry
-from tqdm import tqdm
-
-from .base import oarepo
-
-try:
-    import json5 as json
-except ImportError:
-    import json
-
-from io import StringIO
-
-
-class CheckOk(Exception):
-    pass
-
-
-def dump_data(d):
-    io = StringIO()
-    yaml.safe_dump(d, io, allow_unicode=True)
-    return io.getvalue()
-
-
-@oarepo.command(
-    help="Validate a record. Takes one or two parameters - service name as "
-    "the first one, file name or stdin with record data as the second"
-)
-@click.argument("service-name")
-@click.argument("record-file", required=False)
-@click.option("--community", help="Community name")
-@click.option("--verbose/--no-verbose", is_flag=True)
-@click.option("--with-stacktrace", is_flag=True)
-@click.option(
-    "--fail-on-error",
-    is_flag=True,
-    help="Fail on the first error (for multiple records)",
-)
-@with_appcontext
-def validate(
-    service_name, record_file, community, verbose, with_stacktrace, fail_on_error
-):
-    try:
-        service = current_service_registry.get(service_name)
-    except KeyError:
-        click.secho(f"Service {service_name} not found. Existing services:")
-        for existing in sorted(current_service_registry._services):
-            click.secho(f"    - {existing}")
-        sys.exit(1)
-
-    config = service.config
-    schema = config.schema
-
-    if not record_file:
-        file_content = sys.stdin.read().strip()
-    else:
-        with open(record_file) as f:
-            file_content = f.read()
-
-    if file_content.startswith("{"):
-        data = json.loads(file_content)
-    else:
-        data = list(yaml.safe_load_all(StringIO(file_content)))
-
-    if not isinstance(data, list):
-        data = [data]
-
-    errors_count = 0
-    for idx, d in enumerate(tqdm(data)):
-        if community:
-            d.setdefault("parent", {}).setdefault("communities", {})[
-                "default"
-            ] = community
-        try:
-            loaded = schema().load(d)
-        except Exception as e:
-            click.secho(
-                f"Marshmallow validation of record idx {idx + 1} failed",
-                fg="red",
-            )
-            click.secho(dump_data(d))
-            click.secho(e)
-            if with_stacktrace:
-                traceback.print_exc()
-            if fail_on_error:
-                sys.exit(1)
-            errors_count += 1
-            continue
-
-        if verbose:
-            click.secho(
-                f"Marshmallow validation of record idx {idx+1} has been successful",
-                fg="green",
-            )
-
-        if hasattr(config, "draft_cls"):
-            record_cls = config.draft_cls
-        else:
-            record_cls = config.record_cls
-
-        # Run pre create extensions to check vocabularies
-        try:
-            with db.session.begin_nested():
-
-                rec: Record = record_cls(
-                    loaded, model=record_cls.model_cls(id=None, data=data)
-                )
-                if record_cls.parent_record_cls:
-                    parent = record_cls.parent_record_cls(loaded["parent"])
-                    rec.parent = parent
-
-                for e in rec._extensions:
-                    e.pre_commit(rec)
-                raise CheckOk()
-        except CheckOk:
-            if verbose:
-                click.secho(
-                    f"Pre-commit hook of record idx {idx+1} has been successful",
-                    fg="green",
-                )
-        except Exception as e:
-            click.secho(
-                f"Pre-commit validation of record idx {idx + 1} failed",
-                fg="red",
-            )
-            click.secho(dump_data(d))
-            click.secho(e)
-            if with_stacktrace:
-                traceback.print_exc()
-            if fail_on_error:
-                sys.exit(1)
-            errors_count += 1
-            continue
-
-        if verbose:
-            yaml.safe_dump(loaded, sys.stdout, allow_unicode=True)
-
-    if errors_count:
-        click.secho(f"Validation finished with {errors_count} errors", fg="red")
-        sys.exit(1)
-    else:
-        click.secho("Validation finished successfully", fg="green")
-        sys.exit(0)

oarepo_runtime/datastreams/__init__.py

@@ -1,38 +0,0 @@
-from .asynchronous import AsynchronousDataStream
-from .catalogue import DataStreamCatalogue
-from .datastreams import AbstractDataStream
-from .errors import (
-    DataStreamCatalogueError,
-    DataStreamError,
-    ReaderError,
-    TransformerError,
-    WriterError,
-)
-from .json import JSON, JSONObject
-from .readers import BaseReader
-from .semi_asynchronous import SemiAsynchronousDataStream
-from .synchronous import SynchronousDataStream
-from .transformers import BaseTransformer
-from .types import DataStreamCallback, StreamBatch, StreamEntry
-from .writers import BaseWriter
-
-__all__ = [
-    "JSONObject",
-    "JSON",
-    "StreamEntry",
-    "DataStreamCatalogue",
-    "BaseReader",
-    "BaseWriter",
-    "BaseTransformer",
-    "DataStreamCatalogueError",
-    "ReaderError",
-    "WriterError",
-    "TransformerError",
-    "StreamBatch",
-    "DataStreamError",
-    "DataStreamCallback",
-    "SynchronousDataStream",
-    "AbstractDataStream",
-    "AsynchronousDataStream",
-    "SemiAsynchronousDataStream",
-]

oarepo_runtime/datastreams/asynchronous.py

@@ -1,247 +0,0 @@
-import logging
-from typing import Any, Dict, List, Union
-
-import celery
-from celery.canvas import Signature as CelerySignature
-from celery.canvas import chain
-from celery.result import allow_join_result
-from flask_principal import (
-    ActionNeed,
-    Identity,
-    ItemNeed,
-    Need,
-    RoleNeed,
-    TypeNeed,
-    UserNeed,
-)
-
-from oarepo_runtime.datastreams.datastreams import (
-    AbstractDataStream,
-    DataStreamChain,
-    Signature,
-)
-
-from .datastreams import DataStreamCallback, StreamBatch
-from .json import JSONObject
-from .types import StreamEntryError
-from .writers import BaseWriter
-
-timing = logging.getLogger("oai.harvester.timing")
-log = logging.getLogger("datastreams")
-
-
-class AsynchronousDataStream(AbstractDataStream):
-    def __init__(
-        self,
-        *,
-        readers: List[Union[Signature, Any]],
-        writers: List[Union[Signature, Any]],
-        transformers: List[Union[Signature, Any]] = None,
-        callback: Union[DataStreamCallback, Any],
-        batch_size=100,
-        on_background=True,
-        reader_callback=None,
-    ):
-        super().__init__(
-            readers=readers,
-            writers=writers,
-            transformers=transformers,
-            callback=callback,
-            batch_size=batch_size,
-            reader_callback=reader_callback,
-        )
-        self._on_background = on_background
-
-    def build_chain(self, identity) -> DataStreamChain:
-        return AsynchronousDataStreamChain(
-            transformers=self._transformers,
-            writers=self._writers,
-            on_background=self._on_background,
-            identity=identity,
-        )
-
-    def _reader_error(self, reader, exception):
-        self._callback.apply(
-            kwargs={
-                "callback": f"reader_error",
-                "exception": StreamEntryError.from_exception(exception).json,
-            }
-        )
-
-
-class AsynchronousDataStreamChain(DataStreamChain):
-    def __init__(
-        self,
-        identity: Identity,
-        transformers: List[Signature],
-        writers: List[Signature],
-        on_background=True,
-    ):
-        self._transformers = transformers
-        self._writers = writers
-        self._on_background = on_background
-        self._identity = identity
-
-    def process(self, batch: StreamBatch, callback: CelerySignature):
-        chain = self._prepare_chain(callback)
-        self._call(chain, batch=batch.json)
-
-    def _prepare_chain(self, callback: CelerySignature):
-        chain_def = [
-            datastreams_call_callback.signature(
-                (), kwargs={"callback": callback, "callback_name": "batch_started"}
-            )
-        ]
-        serialized_identity = serialize_identity(self._identity)
-        if self._transformers:
-            for transformer in self._transformers:
-                chain_def.append(
-                    run_datastream_processor.signature(
-                        kwargs={
-                            "processor": transformer.json,
-                            "identity": serialized_identity,
-                            "callback": callback,
-                        }
-                    )
-                )
-
-        for writer in self._writers:
-            chain_def.append(
-                run_datastream_processor.signature(
-                    kwargs={
-                        "processor": writer.json,
-                        "identity": serialized_identity,
-                        "callback": callback,
-                    }
-                )
-            )
-
-        chain_def.append(
-            datastreams_call_callback.signature(
-                (),
-                kwargs=dict(
-                    callback=callback,
-                    callback_name="batch_finished",
-                    identity=serialized_identity,
-                ),
-            )
-        )
-
-        chain_sig = chain(*chain_def)
-        chain_sig.link_error(
-            datastreams_error_callback.signature(
-                (),
-                kwargs=dict(
-                    callback=callback,
-                    callback_name="error",
-                    identity=serialized_identity,
-                ),
-            )
-        )
-        return chain_sig
-
-    def _call(self, sig, **kwargs):
-        if self._on_background:
-            call = sig.apply_async
-        else:
-            call = sig.apply
-        call([], kwargs)
-
-    def finish(self, callback: Signature):
-        "nothing to finish here, dumpers needing finish (such as file dumpers) are not supported in async"
-
-
-@celery.shared_task
-def run_datastream_processor(batch: Dict, *, processor: JSONObject, identity, callback):
-    identity = deserialize_identity(identity)
-    processor_signature = Signature.from_json(processor)
-    deserialized_batch: StreamBatch = StreamBatch.from_json(batch)
-
-    processor = processor_signature.resolve(identity=identity)
-    try:
-        if isinstance(processor, BaseWriter):
-            deserialized_batch = (
-                processor.write(deserialized_batch) or deserialized_batch
-            )
-        else:
-            deserialized_batch = (
-                processor.apply(deserialized_batch) or deserialized_batch
-            )
-
-    except Exception as ex:
-        log.exception("Error processing batch inside %s", processor_signature)
-
-        err = StreamEntryError.from_exception(ex)
-        deserialized_batch.errors.append(err)
-        callback.apply(
-            (),
-            {
-                "batch": deserialized_batch.json,
-                "identity": serialize_identity(identity),
-                "callback": f"{processor_signature.kind.value}_error",
-                "exception": err.json,
-            },
-        )
-    return deserialized_batch.json
-
-
-@celery.shared_task
-def datastreams_call_callback(
-    batch: Dict, *, identity=None, callback, callback_name, **kwargs
-):
-    callback = CelerySignature(callback)
-    callback.apply(
-        kwargs=dict(batch=batch, identity=identity, callback=callback_name, **kwargs)
-    )
-    return batch
-
-
-@celery.shared_task
-def datastreams_error_callback(
-    parent_task_id, *, identity=None, callback, callback_name, **kwargs
-):
-    with allow_join_result():
-        from celery import current_app
-
-        result = current_app.AsyncResult(parent_task_id)
-        result.get(propagate=False)
-
-        callback = CelerySignature(callback)
-        callback.apply(
-            kwargs=dict(
-                batch={},
-                identity=identity,
-                callback=callback_name,
-                result=result.result,
-                traceback=result.traceback,
-                **kwargs,
-            )
-        )
-
-
-def serialize_identity(identity):
-    return {
-        "id": identity.id,
-        "auth_type": identity.auth_type,
-        "provides": [
-            {"type": type(x).__name__, "params": x._asdict()} for x in identity.provides
-        ],
-    }
-
-
-def deserialize_identity(identity_dict):
-    if identity_dict is None:
-        return None
-    ret = Identity(id=identity_dict["id"], auth_type=identity_dict["auth_type"])
-    for provide in identity_dict["provides"]:
-        clz = {
-            "Need": Need,
-            "UserNeed": UserNeed,
-            "RoleNeed": RoleNeed,
-            "TypeNeed": TypeNeed,
-            "ActionNeed": ActionNeed,
-            "ItemNeed": ItemNeed,
-        }[provide["type"]]
-
-        ret.provides.add(clz(**provide["params"]))
-    return ret

oarepo_runtime/datastreams/catalogue.py

@@ -1,150 +0,0 @@
-import dataclasses
-from pathlib import Path
-from typing import Iterator, List
-
-import yaml
-from flask import current_app
-
-from oarepo_runtime.datastreams.datastreams import Signature, SignatureKind
-
-from .errors import DataStreamCatalogueError
-
-
-@dataclasses.dataclass
-class CatalogueDataStream:
-    stream_name: str
-    readers: List[Signature]
-    writers: List[Signature]
-    transformers: List[Signature]
-
-
-class DataStreamCatalogue:
-    def __init__(self, catalogue, content=None) -> None:
-        """
-        Catalogue of data streams. The catalogue contains a dict of:
-        stream_name: stream_definition, where stream definition is an array of:
-
-        - reader: reader_class
-          <rest of parameters go to reader constructor>
-        - transformer: transformer_class
-          <rest of parameters go to transformer constructor>
-        - writer: writer_class
-          <rest of parameters go to writer constructor>
-
-        If reader class is not passed and _source_ is, then the reader class will be taken from the
-        DATASTREAMS_READERS_BY_EXTENSION config variable - map from file extension to reader class.
-
-        If 'service' is passed, service writer will be used with this service
-
-        Transformer class must always be passed.
-        """
-        self._catalogue_path = Path(catalogue)
-        if content:
-            self._catalogue = content
-        else:
-            with open(catalogue) as f:
-                self._catalogue = yaml.safe_load(f)
-
-    @property
-    def path(self):
-        return self._catalogue_path
-
-    @property
-    def directory(self):
-        return self._catalogue_path.parent
-
-    def get_datastreams(self) -> Iterator[CatalogueDataStream]:
-        for stream_name in self._catalogue:
-            yield self.get_datastream(stream_name)
-
-    def __iter__(self):
-        return iter(self._catalogue)
-
-    def get_datastream(
-        self,
-        stream_name,
-    ) -> CatalogueDataStream:
-        stream_definition = self._catalogue[stream_name]
-        readers = []
-        transformers = []
-        writers = []
-        for entry in stream_definition:
-            entry = {**entry}
-            try:
-                if "reader" in entry:
-                    readers.append(
-                        get_signature(
-                            "reader",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "transformer" in entry:
-                    transformers.append(
-                        get_signature(
-                            "transformer",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "writer" in entry:
-                    writers.append(
-                        get_signature(
-                            "writer",
-                            entry,
-                            base_path=str(self.directory),
-                        )
-                    )
-                elif "source" in entry:
-                    readers.append(self.get_reader(entry))
-                elif "service" in entry:
-                    writers.append(self.get_service_writer(entry))
-                else:
-                    raise DataStreamCatalogueError(
-                        "Can not decide what this record is - reader, transformer or service?"
-                    )
-            except DataStreamCatalogueError as e:
-                e.entry = entry
-                e.stream_name = stream_name
-                raise e
-        return CatalogueDataStream(
-            stream_name=stream_name,
-            readers=readers,
-            transformers=transformers,
-            writers=writers,
-        )
-
-    def get_reader(self, entry):
-        entry = {**entry}
-        if not entry.get("reader"):
-            try:
-                source = Path(entry["source"])
-                ext = source.suffix[1:]
-                reader_class = (
-                    current_app.config["DATASTREAMS_READERS_BY_EXTENSION"].get(ext)
-                    or current_app.config["DEFAULT_DATASTREAMS_READERS_BY_EXTENSION"][
-                        ext
-                    ]
-                )
-                entry["reader"] = reader_class
-            except KeyError:
-                raise DataStreamCatalogueError(
-                    f"Do not have loader for file {source} - extension {ext} not defined in DATASTREAMS_READERS_BY_EXTENSION config"
-                )
-        return get_signature(
-            "reader",
-            entry,
-            base_path=str(self.directory),
-        )
-
-    def get_service_writer(self, entry):
-        return Signature(
-            SignatureKind("writer"),
-            "service",
-            kwargs={**entry, "base_path": str(self.directory)},
-        )
-
-
-def get_signature(kind, entry, **kwargs):
-    entry = {**entry, **kwargs}
-    return Signature(kind=SignatureKind(kind), name=entry.pop(kind), kwargs=entry)