oarepo-runtime 1.10.3__py3-none-any.whl → 2.0.0.dev4__py3-none-any.whl

oarepo_runtime-2.0.0.dev4.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[invenio_base.api_apps]
+oarepo_runtime = oarepo_runtime.ext:OARepoRuntime
+
+[invenio_base.apps]
+oarepo_runtime = oarepo_runtime.ext:OARepoRuntime

oarepo_runtime/cli/assets.py

@@ -1,145 +0,0 @@
-import json
-import os
-import re
-from pathlib import Path
-
-import click
-from flask import current_app
-from flask.cli import with_appcontext
-from flask_webpackext import current_webpack
-from importlib_metadata import entry_points
-
-from .base import oarepo
-
-
-@oarepo.group()
-def assets():
-    "OARepo asset addons"
-
-
-@assets.command()
-@click.argument("output_file")
-@click.option("--repository-dir")
-@click.option("--assets-dir", default=".assets")
-@with_appcontext
-def collect(output_file, repository_dir, assets_dir):
-    aliases, asset_dirs, generated_paths = enumerate_assets()
-
-    app_and_blueprints = [current_app] + list(current_app.blueprints.values())
-
-    static_deps = []
-    instance_path = current_app.instance_path
-    if instance_path[-1] != "/":
-        instance_path += "/"
-
-    for bp in app_and_blueprints:
-        if (
-            bp.has_static_folder
-            and os.path.isdir(bp.static_folder)
-            and not bp.static_folder.startswith(instance_path)
-        ):
-            static_deps.append(bp.static_folder)
-
-    root_aliases = {}
-    asset_paths = [Path(x) for x in asset_dirs]
-    for alias, path in aliases.items():
-        for pth in asset_paths:
-            possible_path = pth / path
-            if possible_path.exists():
-                try:
-                    relative_path = str(
-                        possible_path.relative_to(repository_dir or os.getcwd())
-                    )
-                    root_aliases[alias] = "./" + relative_path
-                except ValueError:
-                    root_aliases[alias] = str(Path(assets_dir) / path)
-
-    with open(output_file, "w") as f:
-        json.dump(
-            {
-                "assets": asset_dirs,
-                "static": static_deps,
-                "@aliases": aliases,
-                "@root_aliases": root_aliases,
-                "generated": generated_paths,
-            },
-            f,
-            indent=4,
-            ensure_ascii=False,
-        )
-
-
-def enumerate_assets():
-    asset_dirs = []
-    generated_paths = []
-    aliases = {}
-    themes = current_app.config["APP_THEME"] or ["semantic-ui"]
-    project = current_webpack.project
-    if hasattr(project, 'generated_paths'):
-        generated_paths += project.generated_paths
-
-    for ep in entry_points(group="invenio_assets.webpack"):
-        webpack = ep.load()
-        for wp_theme_name, wp_theme in webpack.themes.items():
-            if wp_theme_name in themes:
-                asset_dirs.append(wp_theme.path)
-                if hasattr(wp_theme, "generated_paths"):
-                    generated_paths += list(set(wp_theme.generated_paths) - set(generated_paths))
-                aliases.update(wp_theme.aliases)
-    return aliases, asset_dirs, generated_paths
-
-
-COMPONENT_LIST_RE = re.compile(
-    r"""
-^
-\s*
-&        # start of import statement & { import "blah"; }
-\s*
-{    
-\s*
-(
-    @import\s+["'](.*?)["']
-    \s*
-    ;
-)+
-\s*
-}""",
-    re.MULTILINE | re.DOTALL | re.VERBOSE,
-)
-
-COMPONENT_RE = re.compile(
-    r"""
-\s*
-@import\s+["'](.*?)["']
-\s*
-;
-\s*
-""",
-    re.MULTILINE | re.DOTALL | re.VERBOSE,
-)
-
-
-
-@assets.command(name="less-components")
-@click.argument("output_file", default="-")
-@with_appcontext
-def less_components(output_file):
-    aliases, asset_dirs, _ = enumerate_assets()
-    asset_dirs = [Path(x) for x in asset_dirs]
-    less_component_files = []
-    for asset_dir in asset_dirs:
-        less_dir = asset_dir / "less"
-        if less_dir.exists():
-            for f in less_dir.glob("**/custom-components.less"):
-                less_component_files.append(f)
-    components = set()
-    for cmp_file in less_component_files:
-        for component_list in COMPONENT_LIST_RE.findall(cmp_file.read_text()):
-            for s in COMPONENT_RE.findall(component_list[0]):
-                components.add(Path(s).stem)
-    data = {"components": list(sorted(components))}
-    if output_file == "-":
-        print(json.dumps(data, indent=4, ensure_ascii=False))
-    else:
-        with open(output_file, "w") as f:
-            json.dump(data, f, indent=4, ensure_ascii=False)

oarepo_runtime/cli/base.py

@@ -1,25 +0,0 @@
-import json
-
-import click
-import importlib_metadata
-
-
-@click.group()
-def oarepo():
-    """OARepo commands."""
-
-
-def as_command(group, name, *args):
-    args = [group.command(name=name), *args]
-    actual = args[-1]
-    for arg in reversed(args[:-1]):
-        actual = arg(actual)
-    return actual
-
-
-@oarepo.command(name="version")
-def get_version():
-    versions = {}
-    for distro in importlib_metadata.distributions():
-        versions[distro.metadata["Name"]] = distro.version
-    print(json.dumps(versions, ensure_ascii=False, indent=4))

oarepo_runtime/cli/cf.py

@@ -1,15 +0,0 @@
-from flask.cli import with_appcontext
-
-from oarepo_runtime.cli import oarepo
-from oarepo_runtime.services.custom_fields.mappings import prepare_cf_indices
-
-
-@oarepo.group()
-def cf():
-    """Custom fields commands."""
-
-
-@cf.command(name="init", help="Prepare custom fields in indices")
-@with_appcontext
-def init():
-    prepare_cf_indices()

oarepo_runtime/cli/check.py

@@ -1,167 +0,0 @@
-import json
-import random
-import traceback
-
-import click
-import kombu.exceptions
-import opensearchpy
-import redis
-from flask import current_app
-from flask.cli import with_appcontext
-from invenio_db import db
-from invenio_files_rest.models import Location
-from invenio_pidstore.models import PersistentIdentifier
-from invenio_records_resources.proxies import current_service_registry
-from opensearchpy import TransportError
-
-from .base import oarepo
-
-
-@oarepo.command(name="check")
-@click.argument("output_file", default="-")
-@with_appcontext
-def check(output_file):
-    status = {}
-    status["db"] = check_database()
-    status["opensearch"] = check_opensearch()
-    status["files"] = check_files()
-    status["mq"] = check_message_queue()
-    status["cache"] = check_cache()
-    if output_file == "-":
-        print(
-            json.dumps(status, indent=4, ensure_ascii=False, default=lambda x: str(x))
-        )
-    else:
-        with open(output_file, "w") as f:
-            json.dump(status, f, ensure_ascii=False, default=lambda x: str(x))
-
-
-def check_database():
-    if not has_database_connection():
-        return "connection_error"
-    try:
-        db.session.begin()
-        try:
-            PersistentIdentifier.query.all()[:1]
-        except:
-            return "not_initialized"
-        alembic = current_app.extensions["invenio-db"].alembic
-        context = alembic.migration_context
-        db_heads = set(context.get_current_heads())
-        source_heads = [x.revision for x in alembic.current()]
-        for h in source_heads:
-            if h not in db_heads:
-                return "migration_pending"
-        return "ok"
-    finally:
-        db.session.rollback()
-
-
-def has_database_connection():
-    try:
-        db.session.begin()
-        db.session.execute("SELECT 1")
-        return True
-    except:
-        return False
-    finally:
-        db.session.rollback()
-
-
-def check_opensearch():
-    services = current_service_registry._services.keys()
-    checked_indexers = set()
-    for service_id in services:
-        service = current_service_registry.get(service_id)
-        record_class = getattr(service.config, "record_cls", None)
-        if not record_class:  # files??
-            continue
-        indexer = getattr(service, "indexer", None)
-        if not indexer:
-            continue
-        if id(indexer) not in checked_indexers:
-            checked_indexers.add(id(indexer))
-            try:
-                indexer.client.indices.exists("test")
-            except opensearchpy.exceptions.ConnectionError:
-                return "connection_error"
-
-
-        try:
-            index = indexer._prepare_index(indexer.record_to_index(record_class))
-        except AttributeError:
-            print(f"Warning: can not get index name for record class {record_class}")
-            continue
-
-        try:
-            service.indexer.client.indices.get(index=index)
-        except TransportError:
-            return f"index_missing:{index}"
-    return "ok"
-
-
-def check_files():
-    if not has_database_connection():
-        return "db_connection_error"
-
-    try:
-        db.session.begin()
-        # check that there is the default location and that is readable
-        default_location = Location.get_default()
-        if not default_location:
-            return "default_location_missing"
-    except:  # NOSONAR - we are not interested what the exception is
-        return "db_error"
-    finally:
-        db.session.rollback()
-
-    try:
-        import s3fs
-    except ImportError:
-        return f"s3_support_not_installed"
-
-    try:
-        info = current_app.extensions["invenio-s3"].init_s3fs_info
-        fs = s3fs.S3FileSystem(default_block_size=4096, **info)
-        fs.ls(default_location.uri.replace("s3://", ""))
-    except:  # NOSONAR - we are not interested what the exception is
-        return f"bucket_does_not_exist:{default_location.uri}"
-
-    return "ok"
-
-
-def check_message_queue():
-    try:
-        from celery import current_app
-
-        current_app.control.inspect().active()
-        return "ok"
-    except kombu.exceptions.OperationalError as e:
-        if isinstance(e.__cause__, ConnectionRefusedError):
-            return "connection_error"
-        return "mq_error"
-    except:  # NOSONAR - we are not interested what the exception is
-        return "mq_error"
-
-
-def check_cache():
-    try:
-        from invenio_cache.proxies import current_cache
-
-        rnd = str(
-            random.randint(0, 10000)  # NOSONAR - this is not a cryptographic random
-        )
-        # it is here just to make sure that what we put to the cache is what we get back
-
-        current_cache.set("oarepo_check", rnd)
-        if current_cache.get("oarepo_check") == rnd:
-            return "ok"
-        else:
-            return "cache_error"
-    except redis.exceptions.ConnectionError as e:
-        if isinstance(e.__cause__, ConnectionRefusedError):
-            return "connection_error"
-        return "cache_exception"
-    except:  # NOSONAR - we are not interested what the exception is
-        traceback.print_exc()
-        return "cache_exception"

oarepo_runtime/cli/configuration.py

@@ -1,51 +0,0 @@
-import json
-from collections.abc import Mapping, Sequence, Set
-
-import click
-from flask import current_app
-from flask.cli import with_appcontext
-from werkzeug.local import LocalProxy
-
-from .base import oarepo
-
-def remove_lazy_objects(obj):
-    if isinstance(obj, Sequence):
-        if isinstance(obj, list):
-            return [remove_lazy_objects(item) for item in obj if not isinstance(item, LocalProxy)]
-        elif isinstance(obj, tuple):
-            return tuple(remove_lazy_objects(item) for item in obj if not isinstance(item, LocalProxy))
-        elif not isinstance(obj, LocalProxy):
-            return obj # strings, bytes, bytesarray etc.
-    elif isinstance(obj, Set):
-        if isinstance(obj, frozenset):
-            return frozenset(remove_lazy_objects(item) for item in obj if not isinstance(item, LocalProxy))
-        return {remove_lazy_objects(item) for item in obj if not isinstance(item, LocalProxy)}
-    elif isinstance(obj, Mapping):
-        return {k: remove_lazy_objects(v) for k, v in obj.items() if not isinstance(v, LocalProxy)}
-    elif not isinstance(obj, LocalProxy):
-        return obj # everything else that is not localproxy
-
-@oarepo.command(name="configuration")
-@click.argument("output_file", default="-")
-@with_appcontext
-def configuration_command(output_file):
-    configuration = remove_lazy_objects(current_app.config)
-
-    try:
-        invenio_db = current_app.extensions["invenio-db"]
-        alembic_config = invenio_db.alembic.config
-        configuration["ALEMBIC_LOCATIONS"] = alembic_config.get_main_option(
-            "version_locations"
-        ).split(",")
-    except Exception as e:
-        configuration["ALEMBIC_LOCATIONS_ERROR"] = str(e)
-
-    if output_file == "-":
-        print(
-            json.dumps(
-                configuration, skipkeys=True, indent=4, ensure_ascii=False, default=lambda x: str(x)
-            )
-        )
-    else:
-        with open(output_file, "w") as f:
-            json.dump(configuration, f,skipkeys=True, ensure_ascii=False, default=lambda x: str(x))

oarepo_runtime/cli/fixtures.py

@@ -1,167 +0,0 @@
-import click
-import tqdm
-from flask import current_app
-from flask.cli import with_appcontext
-from flask_principal import Identity, RoleNeed, UserNeed
-from invenio_access.permissions import any_user, authenticated_user, system_identity
-from invenio_accounts.models import User
-
-from oarepo_runtime.cli import oarepo
-from oarepo_runtime.datastreams import SynchronousDataStream
-from oarepo_runtime.datastreams.asynchronous import AsynchronousDataStream
-from oarepo_runtime.datastreams.fixtures import (
-    FixturesCallback,
-    dump_fixtures,
-    fixtures_asynchronous_callback,
-    load_fixtures,
-)
-from oarepo_runtime.datastreams.types import StatsKeepingDataStreamCallback
-
-
-@oarepo.group()
-def fixtures():
-    """Load and dump fixtures"""
-
-
-@fixtures.command()
-@click.argument("fixture_dir_or_catalogue", required=False)
-@click.option("--include", multiple=True)
-@click.option("--exclude", multiple=True)
-@click.option("--system-fixtures/--no-system-fixtures", default=True, is_flag=True)
-@click.option("--verbose", is_flag=True)
-@click.option("--on-background", is_flag=True)
-@click.option(
-    "--bulk-size",
-    default=100,
-    type=int,
-    help="Size for bulk indexing - this number of records "
-    "will be committed in a single transaction and indexed together",
-)
-@click.option("--batch-size", help="Alias for --bulk-size", type=int)
-@click.option(
-    "--identity", help="Email of the identity that will be used to import the data"
-)
-@with_appcontext
-def load(
-    fixture_dir_or_catalogue=None,
-    include=None,
-    exclude=None,
-    system_fixtures=None,
-    verbose=False,
-    bulk_size=100,
-    on_background=False,
-    batch_size=None,
-    identity=None,
-):
-    """Loads fixtures"""
-    if batch_size:
-        bulk_size = batch_size
-    if not on_background:
-        callback = TQDMCallback(verbose=verbose)
-    else:
-        callback = fixtures_asynchronous_callback.s()
-
-    if fixture_dir_or_catalogue:
-        system_fixtures = False
-
-    if not identity:
-        user = None
-        identity = system_identity
-    else:
-        # identity is user email
-        user = User.query.filter_by(email=identity).one()
-        identity = Identity(user.id)
-
-        # TODO: add provides. How to do it better? It seems that we can not use
-        # flask signals to add these, as they depend on request context that is
-        # not available here
-        identity.provides.add(any_user)
-        identity.provides.add(authenticated_user)
-        identity.provides.add(UserNeed(user.id))
-        for role in getattr(user, "roles", []):
-            identity.provides.add(RoleNeed(role.name))
-        # TODO: community roles ...
-
-    with current_app.wsgi_app.mounts["/api"].app_context():
-        load_fixtures(
-            fixture_dir_or_catalogue,
-            _make_list(include),
-            _make_list(exclude),
-            system_fixtures=system_fixtures,
-            callback=callback,
-            batch_size=bulk_size,
-            datastreams_impl=(
-                AsynchronousDataStream if on_background else SynchronousDataStream
-            ),
-            identity=identity,
-        )
-        if not on_background:
-            _show_stats(callback, "Load fixtures")
-
-
-@fixtures.command()
-@click.option("--include", multiple=True)
-@click.option("--exclude", multiple=True)
-@click.argument("fixture_dir", required=True)
-@click.option("--verbose", is_flag=True)
-@with_appcontext
-def dump(fixture_dir, include, exclude, verbose):
-    """Dump fixtures"""
-    callback = TQDMCallback(verbose=verbose)
-
-    with current_app.wsgi_app.mounts["/api"].app_context():
-        dump_fixtures(
-            fixture_dir,
-            _make_list(include),
-            _make_list(exclude),
-            callback=TQDMCallback(verbose=verbose),
-        )
-        _show_stats(callback, "Dump fixtures")
-
-
-def _make_list(lst):
-    return [
-        item.strip() for lst_item in lst for item in lst_item.split(",") if item.strip()
-    ]
-
-
-def _show_stats(callback: StatsKeepingDataStreamCallback, title: str):
-    print("\n\n")
-    print(f"{title} stats:")
-    print(callback.stats())
-
-
-class TQDMCallback(FixturesCallback):
-    def __init__(self, message_prefix="Loading ", verbose=False):
-        super().__init__()
-        self._tqdm = tqdm.tqdm(unit=" item(s)")
-        self._message_prefix = message_prefix
-        self._verbose = verbose
-
-    def fixture_started(self, fixture_name):
-        self._tqdm.set_description(f"{self._message_prefix}{fixture_name} running")
-
-    def fixture_finished(self, fixture_name):
-        self._tqdm.set_description(f"{self._message_prefix}{fixture_name} finished")
-
-    def batch_finished(self, batch):
-        super().batch_finished(batch)
-        self._tqdm.update(len(batch.entries))
-        for err in batch.errors:
-            self._tqdm.write("Failed batch: {}: {}".format(err, batch))
-        if self._verbose:
-            for entry in batch.entries:
-                if entry.errors:
-                    self._tqdm.write("Failed entry: {}".format(entry))
-
-    def reader_error(self, reader, exception):
-        super().reader_error(reader, exception)
-        self._tqdm.write("Reader error:{}: {}".format(reader, exception))
-
-    def transformer_error(self, batch, transformer, exception):
-        super().transformer_error(batch, transformer, exception)
-        self._tqdm.write("Transformer error: {}: {}".format(transformer, exception))
-
-    def writer_error(self, batch, writer, exception):
-        super().writer_error(batch, writer, exception)
-        self._tqdm.write("Writer error: {}: {}".format(writer, exception))