oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/cli/fixtures.py

@@ -1,167 +0,0 @@
-import click
-import tqdm
-from flask import current_app
-from flask.cli import with_appcontext
-from flask_principal import Identity, RoleNeed, UserNeed
-from invenio_access.permissions import any_user, authenticated_user, system_identity
-from invenio_accounts.models import User
-
-from oarepo_runtime.cli import oarepo
-from oarepo_runtime.datastreams import SynchronousDataStream
-from oarepo_runtime.datastreams.asynchronous import AsynchronousDataStream
-from oarepo_runtime.datastreams.fixtures import (
-    FixturesCallback,
-    dump_fixtures,
-    fixtures_asynchronous_callback,
-    load_fixtures,
-)
-from oarepo_runtime.datastreams.types import StatsKeepingDataStreamCallback
-
-
-@oarepo.group()
-def fixtures():
-    """Load and dump fixtures"""
-
-
-@fixtures.command()
-@click.argument("fixture_dir_or_catalogue", required=False)
-@click.option("--include", multiple=True)
-@click.option("--exclude", multiple=True)
-@click.option("--system-fixtures/--no-system-fixtures", default=True, is_flag=True)
-@click.option("--verbose", is_flag=True)
-@click.option("--on-background", is_flag=True)
-@click.option(
-    "--bulk-size",
-    default=100,
-    type=int,
-    help="Size for bulk indexing - this number of records "
-    "will be committed in a single transaction and indexed together",
-)
-@click.option("--batch-size", help="Alias for --bulk-size", type=int)
-@click.option(
-    "--identity", help="Email of the identity that will be used to import the data"
-)
-@with_appcontext
-def load(
-    fixture_dir_or_catalogue=None,
-    include=None,
-    exclude=None,
-    system_fixtures=None,
-    verbose=False,
-    bulk_size=100,
-    on_background=False,
-    batch_size=None,
-    identity=None,
-):
-    """Loads fixtures"""
-    if batch_size:
-        bulk_size = batch_size
-    if not on_background:
-        callback = TQDMCallback(verbose=verbose)
-    else:
-        callback = fixtures_asynchronous_callback.s()
-
-    if fixture_dir_or_catalogue:
-        system_fixtures = False
-
-    if not identity:
-        user = None
-        identity = system_identity
-    else:
-        # identity is user email
-        user = User.query.filter_by(email=identity).one()
-        identity = Identity(user.id)
-
-        # TODO: add provides. How to do it better? It seems that we can not use
-        # flask signals to add these, as they depend on request context that is
-        # not available here
-        identity.provides.add(any_user)
-        identity.provides.add(authenticated_user)
-        identity.provides.add(UserNeed(user.id))
-        for role in getattr(user, "roles", []):
-            identity.provides.add(RoleNeed(role.name))
-        # TODO: community roles ...
-
-    with current_app.wsgi_app.mounts["/api"].app_context():
-        load_fixtures(
-            fixture_dir_or_catalogue,
-            _make_list(include),
-            _make_list(exclude),
-            system_fixtures=system_fixtures,
-            callback=callback,
-            batch_size=bulk_size,
-            datastreams_impl=(
-                AsynchronousDataStream if on_background else SynchronousDataStream
-            ),
-            identity=identity,
-        )
-        if not on_background:
-            _show_stats(callback, "Load fixtures")
-
-
-@fixtures.command()
-@click.option("--include", multiple=True)
-@click.option("--exclude", multiple=True)
-@click.argument("fixture_dir", required=True)
-@click.option("--verbose", is_flag=True)
-@with_appcontext
-def dump(fixture_dir, include, exclude, verbose):
-    """Dump fixtures"""
-    callback = TQDMCallback(verbose=verbose)
-
-    with current_app.wsgi_app.mounts["/api"].app_context():
-        dump_fixtures(
-            fixture_dir,
-            _make_list(include),
-            _make_list(exclude),
-            callback=TQDMCallback(verbose=verbose),
-        )
-        _show_stats(callback, "Dump fixtures")
-
-
-def _make_list(lst):
-    return [
-        item.strip() for lst_item in lst for item in lst_item.split(",") if item.strip()
-    ]
-
-
-def _show_stats(callback: StatsKeepingDataStreamCallback, title: str):
-    print("\n\n")
-    print(f"{title} stats:")
-    print(callback.stats())
-
-
-class TQDMCallback(FixturesCallback):
-    def __init__(self, message_prefix="Loading ", verbose=False):
-        super().__init__()
-        self._tqdm = tqdm.tqdm(unit=" item(s)")
-        self._message_prefix = message_prefix
-        self._verbose = verbose
-
-    def fixture_started(self, fixture_name):
-        self._tqdm.set_description(f"{self._message_prefix}{fixture_name} running")
-
-    def fixture_finished(self, fixture_name):
-        self._tqdm.set_description(f"{self._message_prefix}{fixture_name} finished")
-
-    def batch_finished(self, batch):
-        super().batch_finished(batch)
-        self._tqdm.update(len(batch.entries))
-        for err in batch.errors:
-            self._tqdm.write("Failed batch: {}: {}".format(err, batch))
-        if self._verbose:
-            for entry in batch.entries:
-                if entry.errors:
-                    self._tqdm.write("Failed entry: {}".format(entry))
-
-    def reader_error(self, reader, exception):
-        super().reader_error(reader, exception)
-        self._tqdm.write("Reader error:{}: {}".format(reader, exception))
-
-    def transformer_error(self, batch, transformer, exception):
-        super().transformer_error(batch, transformer, exception)
-        self._tqdm.write("Transformer error: {}: {}".format(transformer, exception))
-
-    def writer_error(self, batch, writer, exception):
-        super().writer_error(batch, writer, exception)
-        self._tqdm.write("Writer error: {}: {}".format(writer, exception))

oarepo_runtime/cli/index.py

@@ -1,272 +0,0 @@
-import sys
-import traceback
-from io import StringIO
-
-import click
-import yaml
-from flask.cli import with_appcontext
-from invenio_db import db
-from invenio_records_resources.proxies import current_service_registry
-from invenio_search.proxies import current_search
-from werkzeug.utils import ImportStringError, import_string
-
-try:
-    from tqdm import tqdm
-except ImportError:
-
-    def tqdm(generator):
-        yield from generator
-
-
-from .base import oarepo
-
-
-@oarepo.group()
-def index():
-    "OARepo indexing addons"
-
-
-@index.command(
-    help="Create all indices that do not exist yet. "
-    "This is like 'invenio index init' but does not throw "
-    "an exception if some indices already exist"
-)
-@with_appcontext
-def init():
-    click.secho("Creating indexes...", fg="green", bold=True, file=sys.stderr)
-    all_indices = list(gather_all_indices())
-    new_indices = []
-    with click.progressbar(all_indices, label="Checking which indices exist") as bar:
-        for name, alias in bar:
-            if not current_search.client.indices.exists(alias):
-                new_indices.append(name)
-    if new_indices:
-        with click.progressbar(
-            current_search.create(
-                ignore=[400], ignore_existing=True, index_list=new_indices
-            ),
-            length=len(new_indices),
-        ) as bar:
-            for name, response in bar:
-                bar.label = name
-
-
-def gather_all_indices():
-    """Yield index_file, index_name for all indices."""
-
-    # partially copied from invenio-search
-    def _build(tree_or_filename, alias=None):
-        """Build a list of index/alias actions to perform."""
-        for name, value in tree_or_filename.items():
-            if isinstance(value, dict):
-                yield from _build(value, alias=name)
-            else:
-                index_result, alias_result = current_search.create_index(
-                    name, dry_run=True
-                )
-                yield name, alias_result[0]
-
-    yield from _build(current_search.active_aliases)
-
-
-def record_or_service(model):
-    # TODO: is this still used (maybe from somewhere else?)
-    try:
-        service = current_service_registry.get(model)
-    except KeyError:
-        service = None
-    if service and getattr(service, "config", None):
-        record = getattr(service.config, "record_cls", None)
-    else:
-        try:
-            record = import_string(model)
-        except ImportStringError:
-            record = None
-
-    if record is None:
-        click.secho(
-            "Service or model not found. Known services: ",
-            fg="red",
-            bold=True,
-            file=sys.stderr,
-        )
-        for svc in sorted(current_service_registry._services):
-            click.secho(f"    {svc}", file=sys.stderr)
-        sys.exit(1)
-    return record
-
-
-@index.command()
-@with_appcontext
-@click.argument("model", required=False)
-@click.option("--bulk-size", required=False, default=5000, type=int)
-@click.option("--verbose/--no-verbose", default=False)
-def reindex(model, bulk_size, verbose):
-    if not model:
-        services = list(current_service_registry._services.keys())
-    else:
-        services = [model]
-    services = sort_services(services)
-    for service_id in services:
-        click.secho(f"Preparing to index {service_id}", file=sys.stderr)
-
-        try:
-            service = current_service_registry.get(service_id)
-        except KeyError:
-            click.secho(f"Service {service_id} not in known services:", color="red")
-            for known_service_id, known_service in sorted(
-                current_service_registry._services.items()
-            ):
-                click.secho(
-                    f"    {known_service_id} -> {type(known_service).__module__}.{type(known_service).__name__}",
-                    color="red",
-                )
-            sys.exit(1)
-        record_class = getattr(service.config, "record_cls", None)
-
-        id_generators = []
-
-        record_generator = RECORD_GENERATORS.get(service_id, model_records_generator)
-
-        if record_class and hasattr(service, "indexer"):
-            try:
-                id_generators.append(record_generator(record_class))
-            except Exception as e:
-                click.secho(
-                    f"Could not get record ids for {service_id}, exception {e}",
-                    file=sys.stderr,
-                )
-
-        draft_class = getattr(service.config, "draft_cls", None)
-
-        if draft_class and hasattr(service, "indexer"):
-            try:
-                id_generators.append(record_generator(draft_class))
-            except Exception as e:
-                click.secho(
-                    f"Could not get draft record ids for {service_id}, exception {e}",
-                    file=sys.stderr,
-                )
-
-        click.secho(f"Indexing {service_id}", file=sys.stderr)
-        count = 0
-        errors = 0
-        for gen in id_generators:
-            for bulk in generate_bulk_data(gen, service.indexer, bulk_size=bulk_size):
-                index_result = service.indexer.client.bulk(bulk)
-                count += len(bulk) // 2
-                for index_item_result in index_result["items"]:
-                    result = index_item_result["index"]
-                    if result["status"] not in (200, 201):
-                        errors += 1
-                        click.secho(
-                            f"Error indexing record with id {result['_id']}",
-                            fg="red",
-                            file=sys.stderr,
-                        )
-                        click.secho(
-                            dump_yaml(result.get("error")), fg="red", file=sys.stderr
-                        )
-                        if verbose:
-                            click.secho("Record:", file=sys.stderr)
-                            rec = [
-                                bulk[idx + 1]
-                                for idx in range(0, len(bulk), 2)
-                                if bulk[idx]["index"]["_id"] == result["_id"]
-                            ]
-                            if rec:
-                                click.secho(dump_yaml(rec[0]))
-                            else:
-                                click.secho("<no record found>")
-
-        if count:
-            service.indexer.refresh()
-
-        if errors:
-            click.secho(
-                f"Indexing {service_id} failed, indexed {count - errors} records, failed {errors} records.",
-                fg="red",
-                file=sys.stderr,
-            )
-            if not verbose:
-                click.secho("Run with --verbose to see information about the errors")
-        else:
-            click.secho(
-                f"Indexing {service_id} finished, indexed {count} records",
-                fg="green",
-                file=sys.stderr,
-            )
-
-
-def generate_bulk_data(record_generator, record_indexer, bulk_size):
-    data = []
-    n = 0
-    for record in tqdm(record_generator):
-        try:
-            index = record_indexer.record_to_index(record)
-            body = record_indexer._prepare_record(record, index)
-            index = record_indexer._prepare_index(index)
-            data.append(
-                {
-                    "index": {
-                        "_index": index,
-                        "version": record.revision_id,
-                        "version_type": "external_gte",
-                        "_id": body["uuid"],
-                    }
-                }
-            )
-            data.append(body)
-            if len(data) >= bulk_size:
-                yield data
-                data = []
-        except:
-            traceback.print_exc()
-    if data:
-        yield data
-
-
-def dump_yaml(data):
-    io = StringIO()
-    yaml.dump(data, io, allow_unicode=True)
-    return io.getvalue()
-
-
-def model_records_generator(model_class):
-    try:
-        for x in db.session.query(model_class.model_cls.id).filter(
-            model_class.model_cls.is_deleted.is_(False)
-        ):
-            rec_id = x[0]
-            yield model_class.get_record(rec_id)
-    except Exception as e:
-        click.secho(f"Could not index {model_class}: {e}", fg="red", file=sys.stderr)
-
-
-def users_record_generator(model_class):
-    from invenio_accounts.models import User
-    from invenio_users_resources.records.api import UserAggregate
-
-    try:
-        for x in db.session.query(User.id):
-            rec_id = x[0]
-            yield UserAggregate.get_record(rec_id)
-    except Exception as e:
-        click.secho(f"Could not index {model_class}: {e}", fg="red", file=sys.stderr)
-
-
-priorities = ["vocabular", "users", "groups"]
-
-
-def sort_services(services):
-    def idx(x):
-        for idx, p in enumerate(priorities):
-            if p in x:
-                return idx, x
-        return len(priorities), x
-
-    services.sort(key=idx)
-    return services
-
-
-RECORD_GENERATORS = {"users": users_record_generator}

oarepo_runtime/cli/permissions/__init__.py

@@ -1,6 +0,0 @@
-from .base import permissions
-from .evaluate import evaluate_permissions  # noqa
-from .list import list_permissions  # noqa
-from .search import search_permissions  # noqa
-
-__all__ = ["permissions"]

oarepo_runtime/cli/permissions/base.py

@@ -1,26 +0,0 @@
-from flask import current_app
-from flask_principal import Identity, UserNeed, identity_loaded
-from invenio_access.models import User
-
-from ..base import oarepo
-
-
-@oarepo.group()
-def permissions():
-    """Commands for checking and explaining permissions."""
-
-
-def get_user_and_identity(user_id_or_email):
-    try:
-        user_id = int(user_id_or_email)
-        user = User.query.filter_by(id=user_id).one()
-    except ValueError:
-        user = User.query.filter_by(email=user_id_or_email).one()
-
-    identity = Identity(user.id)
-    identity.provides.add(UserNeed(str(user.id)))
-    api_app = current_app.wsgi_app.mounts["/api"]
-    with api_app.app_context():
-        with current_app.test_request_context("/api"):
-            identity_loaded.send(api_app, identity=identity)
-    return user, identity

oarepo_runtime/cli/permissions/evaluate.py

@@ -1,63 +0,0 @@
-import json
-
-import click
-from invenio_records_permissions.policies.records import RecordPermissionPolicy
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.info.permissions.debug import add_debugging
-
-from .base import get_user_and_identity, permissions
-
-
-@permissions.command(name="evaluate")
-@click.argument("user_id_or_email")
-@click.argument("service_name")
-@click.argument("record_id", required=False)
-@click.option("--data", "-d", help="Data to pass to the policy check")
-@click.option("--explain/--no-explain", default=False)
-@click.option("--draft/--published", default=False)
-def evaluate_permissions(
-    user_id_or_email: str,
-    service_name: str,
-    record_id: str | None = None,
-    data: str | None = None,
-    explain: bool = False,
-    draft: bool = False,
-):
-    """Evaluate permissions for a given workflow, community or service."""
-    service = current_service_registry.get(service_name)
-    user, identity = get_user_and_identity(user_id_or_email)
-
-    over = {}
-    if record_id:
-        if draft:
-            over["record"] = service.config.draft_cls.pid.resolve(
-                record_id, registered_only=False
-            )
-        else:
-            over["record"] = service.config.record_cls.pid.resolve(record_id)
-    if data:
-        over["data"] = json.loads(data)
-
-    if explain:
-        over["debug_identity"] = identity
-        add_debugging()
-
-    policy_cls = service.config.permission_policy_cls
-    click.secho(f"Policy: {policy_cls}")
-
-    for action_name in dir(policy_cls):
-        if not action_name.startswith("can_"):
-            continue
-
-        policy: RecordPermissionPolicy = policy_cls(action_name[4:], **over)
-        if explain:
-            click.secho()
-            click.secho(f"## {action_name}")
-        try:
-            if policy.allows(identity):
-                click.secho(f"{action_name}: True", fg="green")
-            else:
-                click.secho(f"{action_name}: False", fg="red")
-        except Exception as e:
-            click.secho(f"{action_name}: {e}", fg="yellow")