oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime/cli/index.py

@@ -1,272 +0,0 @@
-import sys
-import traceback
-from io import StringIO
-
-import click
-import yaml
-from flask.cli import with_appcontext
-from invenio_db import db
-from invenio_records_resources.proxies import current_service_registry
-from invenio_search.proxies import current_search
-from werkzeug.utils import ImportStringError, import_string
-
-try:
-    from tqdm import tqdm
-except ImportError:
-
-    def tqdm(generator):
-        yield from generator
-
-
-from .base import oarepo
-
-
-@oarepo.group()
-def index():
-    "OARepo indexing addons"
-
-
-@index.command(
-    help="Create all indices that do not exist yet. "
-    "This is like 'invenio index init' but does not throw "
-    "an exception if some indices already exist"
-)
-@with_appcontext
-def init():
-    click.secho("Creating indexes...", fg="green", bold=True, file=sys.stderr)
-    all_indices = list(gather_all_indices())
-    new_indices = []
-    with click.progressbar(all_indices, label="Checking which indices exist") as bar:
-        for name, alias in bar:
-            if not current_search.client.indices.exists(alias):
-                new_indices.append(name)
-    if new_indices:
-        with click.progressbar(
-            current_search.create(
-                ignore=[400], ignore_existing=True, index_list=new_indices
-            ),
-            length=len(new_indices),
-        ) as bar:
-            for name, response in bar:
-                bar.label = name
-
-
-def gather_all_indices():
-    """Yield index_file, index_name for all indices."""
-
-    # partially copied from invenio-search
-    def _build(tree_or_filename, alias=None):
-        """Build a list of index/alias actions to perform."""
-        for name, value in tree_or_filename.items():
-            if isinstance(value, dict):
-                yield from _build(value, alias=name)
-            else:
-                index_result, alias_result = current_search.create_index(
-                    name, dry_run=True
-                )
-                yield name, alias_result[0]
-
-    yield from _build(current_search.active_aliases)
-
-
-def record_or_service(model):
-    # TODO: is this still used (maybe from somewhere else?)
-    try:
-        service = current_service_registry.get(model)
-    except KeyError:
-        service = None
-    if service and getattr(service, "config", None):
-        record = getattr(service.config, "record_cls", None)
-    else:
-        try:
-            record = import_string(model)
-        except ImportStringError:
-            record = None
-
-    if record is None:
-        click.secho(
-            "Service or model not found. Known services: ",
-            fg="red",
-            bold=True,
-            file=sys.stderr,
-        )
-        for svc in sorted(current_service_registry._services):
-            click.secho(f"    {svc}", file=sys.stderr)
-        sys.exit(1)
-    return record
-
-
-@index.command()
-@with_appcontext
-@click.argument("model", required=False)
-@click.option("--bulk-size", required=False, default=5000, type=int)
-@click.option("--verbose/--no-verbose", default=False)
-def reindex(model, bulk_size, verbose):
-    if not model:
-        services = list(current_service_registry._services.keys())
-    else:
-        services = [model]
-    services = sort_services(services)
-    for service_id in services:
-        click.secho(f"Preparing to index {service_id}", file=sys.stderr)
-
-        try:
-            service = current_service_registry.get(service_id)
-        except KeyError:
-            click.secho(f"Service {service_id} not in known services:", color="red")
-            for known_service_id, known_service in sorted(
-                current_service_registry._services.items()
-            ):
-                click.secho(
-                    f"    {known_service_id} -> {type(known_service).__module__}.{type(known_service).__name__}",
-                    color="red",
-                )
-            sys.exit(1)
-        record_class = getattr(service.config, "record_cls", None)
-
-        id_generators = []
-
-        record_generator = RECORD_GENERATORS.get(service_id, model_records_generator)
-
-        if record_class and hasattr(service, "indexer"):
-            try:
-                id_generators.append(record_generator(record_class))
-            except Exception as e:
-                click.secho(
-                    f"Could not get record ids for {service_id}, exception {e}",
-                    file=sys.stderr,
-                )
-
-        draft_class = getattr(service.config, "draft_cls", None)
-
-        if draft_class and hasattr(service, "indexer"):
-            try:
-                id_generators.append(record_generator(draft_class))
-            except Exception as e:
-                click.secho(
-                    f"Could not get draft record ids for {service_id}, exception {e}",
-                    file=sys.stderr,
-                )
-
-        click.secho(f"Indexing {service_id}", file=sys.stderr)
-        count = 0
-        errors = 0
-        for gen in id_generators:
-            for bulk in generate_bulk_data(gen, service.indexer, bulk_size=bulk_size):
-                index_result = service.indexer.client.bulk(bulk)
-                count += len(bulk) // 2
-                for index_item_result in index_result["items"]:
-                    result = index_item_result["index"]
-                    if result["status"] not in (200, 201):
-                        errors += 1
-                        click.secho(
-                            f"Error indexing record with id {result['_id']}",
-                            fg="red",
-                            file=sys.stderr,
-                        )
-                        click.secho(
-                            dump_yaml(result.get("error")), fg="red", file=sys.stderr
-                        )
-                        if verbose:
-                            click.secho("Record:", file=sys.stderr)
-                            rec = [
-                                bulk[idx + 1]
-                                for idx in range(0, len(bulk), 2)
-                                if bulk[idx]["index"]["_id"] == result["_id"]
-                            ]
-                            if rec:
-                                click.secho(dump_yaml(rec[0]))
-                            else:
-                                click.secho("<no record found>")
-
-        if count:
-            service.indexer.refresh()
-
-        if errors:
-            click.secho(
-                f"Indexing {service_id} failed, indexed {count - errors} records, failed {errors} records.",
-                fg="red",
-                file=sys.stderr,
-            )
-            if not verbose:
-                click.secho("Run with --verbose to see information about the errors")
-        else:
-            click.secho(
-                f"Indexing {service_id} finished, indexed {count} records",
-                fg="green",
-                file=sys.stderr,
-            )
-
-
-def generate_bulk_data(record_generator, record_indexer, bulk_size):
-    data = []
-    n = 0
-    for record in tqdm(record_generator):
-        try:
-            index = record_indexer.record_to_index(record)
-            body = record_indexer._prepare_record(record, index)
-            index = record_indexer._prepare_index(index)
-            data.append(
-                {
-                    "index": {
-                        "_index": index,
-                        "version": record.revision_id,
-                        "version_type": "external_gte",
-                        "_id": body["uuid"],
-                    }
-                }
-            )
-            data.append(body)
-            if len(data) >= bulk_size:
-                yield data
-                data = []
-        except:
-            traceback.print_exc()
-    if data:
-        yield data
-
-
-def dump_yaml(data):
-    io = StringIO()
-    yaml.dump(data, io, allow_unicode=True)
-    return io.getvalue()
-
-
-def model_records_generator(model_class):
-    try:
-        for x in db.session.query(model_class.model_cls.id).filter(
-            model_class.model_cls.is_deleted.is_(False)
-        ):
-            rec_id = x[0]
-            yield model_class.get_record(rec_id)
-    except Exception as e:
-        click.secho(f"Could not index {model_class}: {e}", fg="red", file=sys.stderr)
-
-
-def users_record_generator(model_class):
-    from invenio_accounts.models import User
-    from invenio_users_resources.records.api import UserAggregate
-
-    try:
-        for x in db.session.query(User.id):
-            rec_id = x[0]
-            yield UserAggregate.get_record(rec_id)
-    except Exception as e:
-        click.secho(f"Could not index {model_class}: {e}", fg="red", file=sys.stderr)
-
-
-priorities = ["vocabular", "users", "groups"]
-
-
-def sort_services(services):
-    def idx(x):
-        for idx, p in enumerate(priorities):
-            if p in x:
-                return idx, x
-        return len(priorities), x
-
-    services.sort(key=idx)
-    return services
-
-
-RECORD_GENERATORS = {"users": users_record_generator}

oarepo_runtime/cli/permissions/__init__.py

@@ -1,6 +0,0 @@
-from .base import permissions
-from .evaluate import evaluate_permissions  # noqa
-from .list import list_permissions  # noqa
-from .search import search_permissions  # noqa
-
-__all__ = ["permissions"]

oarepo_runtime/cli/permissions/base.py

@@ -1,26 +0,0 @@
-from flask import current_app
-from flask_principal import Identity, UserNeed, identity_loaded
-from invenio_access.models import User
-
-from ..base import oarepo
-
-
-@oarepo.group()
-def permissions():
-    """Commands for checking and explaining permissions."""
-
-
-def get_user_and_identity(user_id_or_email):
-    try:
-        user_id = int(user_id_or_email)
-        user = User.query.filter_by(id=user_id).one()
-    except ValueError:
-        user = User.query.filter_by(email=user_id_or_email).one()
-
-    identity = Identity(user.id)
-    identity.provides.add(UserNeed(str(user.id)))
-    api_app = current_app.wsgi_app.mounts["/api"]
-    with api_app.app_context():
-        with current_app.test_request_context("/api"):
-            identity_loaded.send(api_app, identity=identity)
-    return user, identity

oarepo_runtime/cli/permissions/evaluate.py

@@ -1,63 +0,0 @@
-import json
-
-import click
-from invenio_records_permissions.policies.records import RecordPermissionPolicy
-from invenio_records_resources.proxies import current_service_registry
-
-from oarepo_runtime.info.permissions.debug import add_debugging
-
-from .base import get_user_and_identity, permissions
-
-
-@permissions.command(name="evaluate")
-@click.argument("user_id_or_email")
-@click.argument("service_name")
-@click.argument("record_id", required=False)
-@click.option("--data", "-d", help="Data to pass to the policy check")
-@click.option("--explain/--no-explain", default=False)
-@click.option("--draft/--published", default=False)
-def evaluate_permissions(
-    user_id_or_email: str,
-    service_name: str,
-    record_id: str | None = None,
-    data: str | None = None,
-    explain: bool = False,
-    draft: bool = False,
-):
-    """Evaluate permissions for a given workflow, community or service."""
-    service = current_service_registry.get(service_name)
-    user, identity = get_user_and_identity(user_id_or_email)
-
-    over = {}
-    if record_id:
-        if draft:
-            over["record"] = service.config.draft_cls.pid.resolve(
-                record_id, registered_only=False
-            )
-        else:
-            over["record"] = service.config.record_cls.pid.resolve(record_id)
-    if data:
-        over["data"] = json.loads(data)
-
-    if explain:
-        over["debug_identity"] = identity
-        add_debugging()
-
-    policy_cls = service.config.permission_policy_cls
-    click.secho(f"Policy: {policy_cls}")
-
-    for action_name in dir(policy_cls):
-        if not action_name.startswith("can_"):
-            continue
-
-        policy: RecordPermissionPolicy = policy_cls(action_name[4:], **over)
-        if explain:
-            click.secho()
-            click.secho(f"## {action_name}")
-        try:
-            if policy.allows(identity):
-                click.secho(f"{action_name}: True", fg="green")
-            else:
-                click.secho(f"{action_name}: False", fg="red")
-        except Exception as e:
-            click.secho(f"{action_name}: {e}", fg="yellow")

oarepo_runtime/cli/permissions/list.py

@@ -1,239 +0,0 @@
-from typing import get_type_hints
-
-import click
-
-from oarepo_runtime.info.permissions.debug import add_debugging
-
-from .base import permissions
-
-
-@permissions.command(name="list")
-@click.option("--workflow", "-w", help="Workflow name")
-@click.option("--community", "-c", help="Community name")
-@click.option("--service", "-s", help="Service name")
-def list_permissions(workflow, community, service):
-    """List all permissions for a given workflow, community or service."""
-    # intentionally import here to enable oarepo-runtime to be used
-    # without oarepo-workflows and oarepo-communities
-    from invenio_communities.communities.records.api import Community
-    from invenio_communities.communities.records.models import CommunityMetadata
-    from oarepo_workflows.proxies import current_oarepo_workflows
-
-    if workflow:
-        wf = current_oarepo_workflows.record_workflows[workflow]
-        permission_policy = wf.permission_policy_cls
-    elif community:
-        community_db = CommunityMetadata.query.filter_by(slug=community).one()
-        community_obj = Community(community_db.data, model=community_db)
-        workflow = community_obj["custom_fields"].get("workflow", "default")
-        wf = current_oarepo_workflows.record_workflows[workflow]
-        permission_policy = wf.permission_policy_cls
-    elif service:
-        from invenio_records_resources.proxies import current_service_registry
-
-        swc = current_service_registry.get(service)
-        permission_policy = swc.config.permission_policy_cls
-    else:
-        raise click.UsageError(
-            "You must specify either --workflow, --community or --service."
-        )
-
-    add_debugging()
-
-    p = permission_policy("read")
-
-    for action_name in dir(permission_policy):
-        if not action_name.startswith("can_"):
-            continue
-        action_permission_generators = getattr(p, action_name)
-        debugs = []
-        for x in action_permission_generators:
-            debugs.append(x.to_debug_dict())
-
-        print("")
-        print(f"## {action_name}")
-        print(get_permission_documentation(permission_policy, action_name))
-        for perm in debugs:
-            print_permission_markdown(perm)
-
-
-def is_simple_dict(d):
-    return all(isinstance(v, (str, int, float, bool)) for v in d.values())
-
-
-def format_simple_dict_values(d):
-    # returns k=v, k=v, ...
-    return ", ".join(f"{k}={v}" for k, v in d.items())
-
-
-def print_permission_markdown(p, level=0):
-    for k, v in p.items():
-        prologue = "  " * level + f"- {k}"
-        if v is None or v == {} or v == []:
-            print(prologue)
-            continue
-        elif isinstance(v, dict):
-            if is_simple_dict(v):
-                print(f"{prologue}: {format_simple_dict_values(v)}")
-            else:
-                print(prologue)
-                print_permission_markdown(v, level + 1)
-        elif isinstance(v, list):
-            print(prologue)
-            for x in v:
-                print_permission_markdown(x, level + 1)
-        else:
-            print("  " * (level + 1) + f"{v}")
-
-
-def get_permission_documentation(policy, permission_can_name):
-    type_hints = get_type_hints(policy, include_extras=True)
-    annotation = type_hints.get(permission_can_name)
-    if annotation:
-        for md in annotation.__metadata__:
-            if isinstance(md, str):
-                return md
-    ret = default_permissions_documentation.get(permission_can_name, "")
-    return "\n".join(x.strip() for x in ret.split("\n"))
-
-
-default_permissions_documentation = {
-    # records
-    "can_create": """
-        Grants users the ability to create new records in the 
-        repository, often assigned to roles like submitters, owners, or curators.
-        The result of this action is typically a draft record.
-    """,
-    "can_read": """
-        Allows users to view or access records, with access 
-        possibly dependent on the record's state (e.g., draft or published) and the 
-        user's role.
-    """,
-    "can_read_deleted": """
-        Permission to view or access records, including soft-deleted ones. This 
-        permission is used to filter search results. It should include an 
-        `IfRecordDeleted` permission generator, which grants access to deleted 
-        records to a subset of users (e.g., owners, curators, etc.). For 
-        `service.read`, this permission is applied when record is deleted and 
-        `include_deleted` is passed; otherwise, a `RecordDeletedException` is raised.
-    """,
-    "can_search": """
-        Grants users the ability to search for records within the 
-        repository, typically available to all users. The search results are
-        filtered based on the can_read_deleted permission for published records (/api/datasets)
-        and the can_read_draft permission for draft records (/api/user/datasets).
-    """,
-    "can_update": """
-        Grants users the ability to update or modify published records.
-        In NRP repositories, this is normally disabled as updates are performed
-        on draft records, which are then published.
-    """,
-    "can_delete": """
-        Grants users the ability to delete records, often restricted 
-        to owners, curators, or specific roles, and dependent on the record's 
-        state (e.g., draft). For published records, the deletion is performed
-        via a specialized request, not directly.
-    """,
-    "can_new_version": """
-        Allows users to create a new version of a record. In NRP,
-        this is not used directly but always via a request, which is mostly
-        auto-approved.
-    """,
-    "can_edit": """
-        Grants users the ability to modify the metadata of a record. In NRP, this
-        is not used directly but always via a request, which is mostly auto-approved.
-    """,
-    "can_manage": """
-        Grants users the ability to manage records. Currently it is used just
-        for reindexing records with latest version first (reindex_latest_first)
-        service call, not mapped to REST API.
-    """,
-    "can_manage_record_access": """
-        Grants users the ability to manage record access.
-    """,
-    # draft records
-    "can_read_draft": """
-        Enables users to view or access draft records, typically equivalent to 
-        the general 'can_read' permission but specific to drafts.
-    """,
-    "can_delete_draft": """
-        Allows users to delete draft records, typically equivalent to the general 
-        'can_delete' permission but specific to drafts.
-    """,
-    "can_update_draft": """
-        Allows users to update draft records. It is typically granted to record
-        owner, but can be restricted by the record status (such as records submitted
-        to review being locked).
-    """,
-    "can_publish": """
-        Grants users the ability to publish records, making them publicly 
-        available or finalizing their state. In NRP repositories, this is
-        not used directly but always via a request.
-    """,
-    # files
-    "can_read_files": """
-        Grants users the ability to view or access file metadata associated with records, 
-        with access dependent on the record's state and the user's role.
-    """,
-    "can_create_files": """
-        Enables users to create new files within published records in the repository.
-        Normally disabled as files are created in draft records and then published.
-    """,
-    "can_delete_files": """
-        Enables users to delete files associated with published records in the repository.
-        Normally disabled as files can not be deleted from published records.
-    """,
-    "can_update_files": """
-        Enables users to update or modify files within published records in the repository.
-        Normally disabled as files are updated in draft records and then published.
-    """,
-    "can_commit_files": """
-        Allows users to finalize file upload to published records in the repository.
-        Normally disabled as files are uploaded to draft records and then published.
-    """,
-    "can_get_content_files": """
-        Allows users to access or retrieve the content of files on records published 
-        in the repository, with access depending on the record's state and the user's role.
-    """,
-    "can_manage_files": """
-        Grants users the ability to manage files (that is, change if the files are
-        required on the record or not).
-    """,
-    "can_read_deleted_files": """
-        Allows users to view or access files associated with deleted records, 
-        often restricted to specific conditions.
-    """,
-    "can_set_content_files": """
-        Enables users to upload files to published records. Normally disabled as files
-        are uploaded to draft records and then published.
-    """,
-    # draft files
-    "can_draft_read_files": """
-        Allows users to view or access file metadata associated with draft records,
-        typically equivalent to the general 'can_read_files' permission but specific
-        to drafts.
-    """,
-    "can_draft_create_files": """
-        Allows users to create files specifically for draft records.
-    """,
-    "can_draft_get_content_files": """
-        Allows users to access or retrieve the content of files on draft records.
-    """,
-    "can_draft_set_content_files": """
-        Allows users to upload files to draft records.
-    """,
-    "can_draft_commit_files": """
-        Allows users to finalize file upload to draft records.
-    """,
-    "can_draft_update_files": """
-        Allows users to update or modify files within draft records.
-    """,
-    "can_draft_delete_files": """
-        Allows users to delete files associated with draft records.
-    """,
-    # misc
-    "can_create_or_update_many": """
-        Allows bulk creation or updating of multiple records or files.
-        Not used at the moment.
-    """,
-}