oarepo-runtime 1.5.6__tar.gz → 1.5.7__tar.gz

{oarepo-runtime-1.5.6/oarepo_runtime.egg-info → oarepo-runtime-1.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oarepo-runtime
-Version: 1.5.6
+Version: 1.5.7
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/cli/fixtures.py

@@ -48,6 +48,9 @@ def load(
     else:
         callback = fixtures_asynchronous_callback.s()
 
+    if fixture_dir:
+        system_fixtures = False
+
     with current_app.wsgi_app.mounts["/api"].app_context():
         load_fixtures(
             fixture_dir,
@@ -56,9 +59,9 @@ def load(
             system_fixtures=system_fixtures,
             callback=callback,
             batch_size=bulk_size,
-            datastreams_impl=AsynchronousDataStream
-            if on_background
-            else SynchronousDataStream,
+            datastreams_impl=(
+                AsynchronousDataStream if on_background else SynchronousDataStream
+            ),
         )
         if not on_background:
             _show_stats(callback, "Load fixtures")

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/datastreams/fixtures.py

@@ -57,12 +57,20 @@ def load_fixtures(
         )
 
     if system_fixtures:
-        for r in reversed(
-            sorted(
-                pkg_resources.iter_entry_points("oarepo.fixtures"), key=lambda r: r.name
-            )
-        ):
-            pkg = r.load()
+
+        def get_priority(name):
+            match = re.match(r"(\d+)-", name)
+            if match:
+                return -int(match.group(1))
+            return 0
+
+        entry_points = list(
+            (get_priority(r.name), r.name, r)
+            for r in pkg_resources.iter_entry_points("oarepo.fixtures")
+        )
+        entry_points.sort(key=lambda x: x[:2])
+        for r in entry_points:
+            pkg = r[2].load()
             pkg_fixture_dir = Path(pkg.__file__)
             if pkg_fixture_dir.is_file():
                 pkg_fixture_dir = pkg_fixture_dir.parent
@@ -90,6 +98,7 @@ def _load_fixtures_from_catalogue(
             continue
         if any(x.match(catalogue_datastream.stream_name) for x in exclude):
             continue
+
         fixtures.add(catalogue_datastream.stream_name)
 
         datastream = datastreams_impl(

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/ext.py

@@ -28,9 +28,9 @@ class OARepoRuntime(object):
 
         for k in ext_config.OAREPO_PERMISSIONS_PRESETS:
             if k not in app.config["OAREPO_PERMISSIONS_PRESETS"]:
-                app.config["OAREPO_PERMISSIONS_PRESETS"][
-                    k
-                ] = ext_config.OAREPO_PERMISSIONS_PRESETS[k]
+                app.config["OAREPO_PERMISSIONS_PRESETS"][k] = (
+                    ext_config.OAREPO_PERMISSIONS_PRESETS[k]
+                )
 
         for k in dir(ext_config):
             if k == "DEFAULT_DATASTREAMS_EXCLUDES":
@@ -53,5 +53,3 @@ class OARepoRuntime(object):
         for val_k, val_value in source.items():
             if val_k not in target:
                 target[val_k] = val_value
-
-

oarepo-runtime-1.5.7/oarepo_runtime/info/views.py

@@ -0,0 +1,302 @@
+import json
+import logging
+import re
+from functools import cached_property
+from urllib.parse import urljoin
+
+import importlib_metadata
+import importlib_resources
+import marshmallow as ma
+from flask import current_app, request, url_for
+from flask_resources import (
+    Resource,
+    ResourceConfig,
+    from_conf,
+    request_parser,
+    resource_requestctx,
+    response_handler,
+    route,
+)
+from flask_restful import abort
+from invenio_base.utils import obj_or_import_string
+from invenio_jsonschemas import current_jsonschemas
+from invenio_records_resources.proxies import current_service_registry
+
+logger = logging.getLogger("oarepo_runtime.info")
+
+
+class InfoConfig(ResourceConfig):
+    blueprint_name = "oarepo_runtime_info"
+    url_prefix = "/.well-known/repository"
+
+    schema_view_args = {"schema": ma.fields.Str()}
+    model_view_args = {"model": ma.fields.Str()}
+
+    def __init__(self, app):
+        self.app = app
+
+    @cached_property
+    def components(self):
+        return tuple(
+            obj_or_import_string(x)
+            for x in self.app.config.get("INFO_ENDPOINT_COMPONENTS", [])
+        )
+
+
+schema_view_args = request_parser(from_conf("schema_view_args"), location="view_args")
+model_view_args = request_parser(from_conf("model_view_args"), location="view_args")
+
+
+class InfoResource(Resource):
+    def create_url_rules(self):
+        return [
+            route("GET", "/", self.repository),
+            route("GET", "/models", self.models),
+            route("GET", "/schema/<path:schema>", self.schema),
+            route("GET", "/models/<model>", self.model),
+        ]
+
+    @cached_property
+    def components(self):
+        return [x(self) for x in self.config.components]
+
+    @response_handler()
+    def repository(self):
+        """Repository endpoint."""
+        ret = {
+            "name": current_app.config.get("THEME_SITENAME", ""),
+            "description": current_app.config.get("REPOSITORY_DESCRIPTION", ""),
+            "version": get_package_version("repo"),
+            "invenio_version": get_package_version("oarepo"),
+            "transfers": [
+                "local-file",
+                "url-fetch",
+                # TODO: where to get these? (permissions?)
+                # "direct-s3",
+            ],
+            "links": {
+                "self": url_for(request.endpoint, _external=True),
+                "models": url_for("oarepo_runtime_info.models", _external=True),
+            },
+        }
+        self.call_components("repository", data=ret)
+        return ret, 200
+
+    @response_handler(many=True)
+    def models(self):
+        data = []
+        # iterate entrypoint oarepo.models
+        for model in importlib_metadata.entry_points().select(group="oarepo.models"):
+            package_name, file_name = model.value.split(":")
+            model_data = json.loads(
+                importlib_resources.files(package_name).joinpath(file_name).read_text()
+            )
+            model_data = model_data["model"]
+            if model_data["type"] != "model":
+                continue
+
+            service = self._get_service(model_data)
+            if not service:
+                continue
+
+            model_features = self._get_model_features(model_data)
+
+            links = {
+                "api": self._get_model_api_endpoint(model_data),
+                "html": self._get_model_html_endpoint(model_data),
+                "schema": self._get_model_schema_endpoint(model_data),
+                "model": self._get_model_model_endpoint(model.name),
+                # "openapi": url_for(self._get_model_openapi_endpoint(model_data), _external=True)
+            }
+
+            links["published"] = links["api"]
+            if "drafts" in model_features:
+                links["drafts"] = self._get_model_draft_endpoint(model_data)
+
+            data.append(
+                {
+                    "name": model_data.get(
+                        "model-name", model_data.get("module", {}).get("base", "")
+                    ).lower(),
+                    "description": model_data.get("model-description", ""),
+                    "version": model_data["json-schema-settings"]["version"],
+                    "features": model_features,
+                    "links": links,
+                    # TODO: we also need to get previous schema versions here if we support
+                    # multiple version of the same schema at the same time
+                    "schemas": self._get_model_schemas(service),
+                }
+            )
+        self.call_components("repository", data=data)
+        return data, 200
+
+    @schema_view_args
+    @response_handler()
+    def schema(self):
+        schema = resource_requestctx.view_args["schema"]
+        return current_jsonschemas.get_schema(schema, resolved=True), 200
+
+    @model_view_args
+    @response_handler()
+    def model(self):
+        model = resource_requestctx.view_args["model"]
+        for _model in importlib_metadata.entry_points().select(
+            group="oarepo.models", name=model
+        ):
+            package_name, file_name = _model.value.split(":")
+            model_data = json.loads(
+                importlib_resources.files(package_name).joinpath(file_name).read_text()
+            )
+            return self._remove_implementation_details_from_model(model_data), 200
+        abort(404)
+
+    IMPLEMENTATION_DETAILS = re.compile(
+        r"""
+^(
+  class | 
+  .*-class |
+  base-classes |
+  .*-base-classes |
+  module |
+  generate |
+  imports |
+  extra-code |
+  components |
+  .*-args
+)$
+    """,
+        re.VERBOSE,
+    )
+
+    def _remove_implementation_details_from_model(self, model):
+        if isinstance(model, dict):
+            return self._remove_implementation_details_from_model_dict(model)
+        elif isinstance(model, list):
+            return self._remove_implementation_details_from_model_list(model)
+        else:
+            return model
+
+    def _remove_implementation_details_from_model_dict(self, model):
+        ret = {}
+        for k, v in model.items():
+            if not self.IMPLEMENTATION_DETAILS.match(k):
+                new_value = self._remove_implementation_details_from_model(v)
+                if new_value is not None and new_value != {} and new_value != []:
+                    ret[k] = new_value
+        return ret
+
+    def _remove_implementation_details_from_model_list(self, model):
+        ret = []
+        for v in model:
+            new_value = self._remove_implementation_details_from_model(v)
+            if new_value is not None and new_value != {} and new_value != []:
+                ret.append(new_value)
+        return ret
+
+    def call_components(self, method_name, **kwargs):
+        for component in self.config.components:
+            if hasattr(component, method_name):
+                getattr(component, method_name)(**kwargs)
+
+    def _get_model_features(self, model):
+        features = []
+        if model.get("requests", {}):
+            features.append("requests")
+        if model.get("draft", {}):
+            features.append("drafts")
+        if model.get("files", {}):
+            features.append("files")
+        return features
+
+    def _get_model_api_endpoint(self, model):
+        try:
+            alias = model["api-blueprint"]["alias"]
+            return api_url_for(f"{alias}.search", _external=True)
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model api endpoint")
+            return None
+
+    def _get_model_draft_endpoint(self, model):
+        try:
+            alias = model["api-blueprint"]["alias"]
+            return api_url_for(f"{alias}.search_user_records", _external=True)
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model draft endpoint")
+            return None
+
+    def _get_model_html_endpoint(self, model):
+        try:
+            return urljoin(
+                self._get_model_api_endpoint(model),
+                model["resource-config"]["base-html-url"],
+            )
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model html endpoint")
+            return None
+
+    def _get_model_schema_endpoint(self, model):
+        try:
+            return url_for(
+                "oarepo_runtime_info.schema",
+                schema=model["json-schema-settings"]["name"],
+                _external=True,
+            )
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model schema endpoint")
+            return None
+
+    def _get_model_model_endpoint(self, model):
+        try:
+            return url_for("oarepo_runtime_info.model", model=model, _external=True)
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model model endpoint")
+            return None
+
+    def _get_model_schemas(self, service):
+        try:
+            record_cls = service.config.record_cls
+            schema = getattr(record_cls, "schema", None)
+            if schema is not None:
+                return [schema.value]
+        except:  # NOSONAR noqa
+            logger.exception("Failed to get model schemas")
+        return []
+
+    def _get_service(self, model_data):
+        service_id = model_data["service-config"]["service-id"]
+        try:
+            service = current_service_registry.get(service_id)
+        except KeyError:
+            return None
+        return service
+
+
+def create_wellknown_blueprint(app):
+    """Create blueprint."""
+    config_class = obj_or_import_string(
+        app.config.get("INFO_ENDPOINT_CONFIG", InfoConfig)
+    )
+    return InfoResource(config=config_class(app)).as_blueprint()
+
+
+def get_package_version(package_name):
+    """Get package version."""
+    from pkg_resources import get_distribution
+
+    try:
+        return re.sub(r"\+.*", "", get_distribution(package_name).version)
+    except Exception:  # NOSONAR noqa
+        logger.exception(f"Failed to get package version for {package_name}")
+        return None
+
+
+def api_url_for(endpoint, _external=True, **values):
+    """API url_for."""
+    site_api_url = current_app.config["SITE_API_URL"]
+    site_url = current_app.config["SITE_UI_URL"]
+    base_url = url_for(endpoint, **values, _external=_external)
+    if base_url.startswith(site_api_url):
+        return base_url
+    if base_url.startswith(site_url):
+        return base_url.replace(site_url, site_api_url)
+    raise ValueError(f"URL {base_url} does not start with {site_url} or {site_api_url}")

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/records/systemfields/mapping.py

@@ -12,6 +12,10 @@ class MappingSystemFieldMixin:
     def mapping_settings(self):
         return {}
 
+    @property
+    def dynamic_templates(self):
+        return []
+
 
 class SystemFieldDumperExt(SearchDumperExt):
     def dump(self, record, data):

oarepo-runtime-1.5.7/oarepo_runtime/records/systemfields/synthetic.py

@@ -0,0 +1,79 @@
+from invenio_records.systemfields import SystemField
+
+from .mapping import MappingSystemFieldMixin
+
+
+class SyntheticSystemField(MappingSystemFieldMixin, SystemField):
+    """
+        A class that provides a synthetic system field, that is a system field that
+        generates its content from what is already present inside the record.
+
+        The field is not stored in the record, but is generated on the fly when
+        the record is being indexed.
+
+        Usage:
+
+        1. Create a new class that inherits from SyntheticSystemField
+        2. Implement the _value method that returns the value of the field from a data (
+           either a dictionary or an instance of the record class)
+        3. Put the class onto the record. If you use oarepo-model-builder, add it to the model
+           like:
+           ```yaml
+    record:
+      record:
+        extra-code: |-2
+              # extra custom fields for facets
+              faculty = {{common.theses.synthetic_fields.FacultySystemField}}()
+              department = {{common.theses.synthetic_fields.DepartmentSystemField}}()
+              defenseYear = {{common.theses.synthetic_fields.DefenseYearSystemField}}()
+           ```
+
+        4. Add the extra fields to the mapping and facets. If using oarepo-model-builder, add it to the
+           model like the following piece of code and compile the model:
+           ```yaml
+    record:
+      properties:
+        faculty:
+          type: vocabulary
+          vocabulary-type: institutions
+          facets:
+            facet-groups:
+            - default
+          label.cs: Fakulta
+          label.en: Faculty
+
+
+        department:
+          type: vocabulary
+          vocabulary-type: institutions
+          facets:
+            facet-groups:
+            - default
+          label.cs: Ústav
+          label.en: Department
+
+        defenseYear:
+          type: integer
+          facets:
+            facet-groups:
+            - default
+          label.cs: Rok obhajoby
+          label.en: Defense year
+           ```
+    """
+
+    def search_dump(self, data):
+        dt = self._value(data)
+        if dt:
+            data[self.key] = dt
+
+    def search_load(self, data):
+        data.pop(self.key)
+
+    def __get__(self, record, owner=None):
+        if record is None:
+            return self
+        return self._value(record)
+
+    def _value(self, data):
+        raise NotImplementedError("You must implement the _value method")

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/services/custom_fields/mappings.py

@@ -80,6 +80,7 @@ def prepare_cf_index(config: RecordServiceConfig):
         # get mapping
         mapping = fld.mapping
         settings = fld.mapping_settings
+        dynamic_templates = fld.dynamic_templates
 
         # upload mapping
         try:
@@ -98,20 +99,25 @@ def prepare_cf_index(config: RecordServiceConfig):
                     ),
                     using=current_search_client,
                 )
-                update_index(draft_index, settings, mapping)
+                update_index(draft_index, settings, mapping, dynamic_templates)
 
         except search.RequestError as e:
             click.secho("An error occurred while creating custom fields.", fg="red")
             click.secho(e.info["error"]["reason"], fg="red")
 
 
-def update_index(record_index, settings, mapping):
+def update_index(record_index, settings, mapping, dynamic_templates=None):
     if settings:
         record_index.close()
         record_index.put_settings(body=settings)
         record_index.open()
+    body = {}
     if mapping:
-        record_index.put_mapping(body={"properties": mapping})
+        body["properties"] = mapping
+    if dynamic_templates:
+        body["dynamic_templates"] = dynamic_templates
+    if body:
+        record_index.put_mapping(body=body)
 
 
 def get_mapping_fields(record_class) -> Iterable[MappingSystemFieldMixin]:

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime/services/results.py

@@ -1,4 +1,3 @@
-import inspect
 
 from invenio_records_resources.services.records.results import (
     RecordItem as BaseRecordItem,
@@ -7,9 +6,12 @@ from invenio_records_resources.services.records.results import (
     RecordList as BaseRecordList,
 )
 
+
 class RecordItem(BaseRecordItem):
     """Single record result."""
+
     components = []
+
     @property
     def data(self):
         if self._data:
@@ -19,8 +21,10 @@ class RecordItem(BaseRecordItem):
             c.update_data(self._identity, self._record, _data)
         return _data
 
+
 class RecordList(BaseRecordList):
     components = []
+
     @property
     def hits(self):
         """Iterator over the hits."""

oarepo-runtime-1.5.7/oarepo_runtime/utils/functools.py

@@ -0,0 +1,8 @@
+def try_sequence(*funcs, ignored_exceptions=(), raised_exception=Exception):
+    raised_exceptions = []
+    for func in funcs:
+        try:
+            return func()
+        except ignored_exceptions as e:
+            raised_exceptions.append(e)
+    raise raised_exception(raised_exceptions) from raised_exceptions[-1]

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7/oarepo_runtime.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oarepo-runtime
-Version: 1.5.6
+Version: 1.5.7
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime.egg-info/SOURCES.txt

@@ -52,6 +52,8 @@ oarepo_runtime/datastreams/writers/utils.py
 oarepo_runtime/datastreams/writers/validation_errors.py
 oarepo_runtime/datastreams/writers/yaml.py
 oarepo_runtime/i18n/__init__.py
+oarepo_runtime/info/__init__.py
+oarepo_runtime/info/views.py
 oarepo_runtime/records/__init__.py
 oarepo_runtime/records/dumpers/__init__.py
 oarepo_runtime/records/dumpers/edtf_interval.py
@@ -69,6 +71,7 @@ oarepo_runtime/records/systemfields/has_draftcheck.py
 oarepo_runtime/records/systemfields/icu.py
 oarepo_runtime/records/systemfields/mapping.py
 oarepo_runtime/records/systemfields/record_status.py
+oarepo_runtime/records/systemfields/synthetic.py
 oarepo_runtime/resources/__init__.py
 oarepo_runtime/resources/file_resource.py
 oarepo_runtime/resources/localized_ui_json_serializer.py
@@ -114,5 +117,6 @@ oarepo_runtime/translations/cs/LC_MESSAGES/messages.po
 oarepo_runtime/translations/en/LC_MESSAGES/messages.mo
 oarepo_runtime/translations/en/LC_MESSAGES/messages.po
 oarepo_runtime/utils/__init__.py
+oarepo_runtime/utils/functools.py
 oarepo_runtime/utils/path.py
 tests/pkg_data/__init__.py

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/oarepo_runtime.egg-info/entry_points.txt

@@ -4,5 +4,8 @@ oarepo_runtime = oarepo_runtime.ext:OARepoRuntime
 [invenio_base.apps]
 oarepo_runtime = oarepo_runtime.ext:OARepoRuntime
 
+[invenio_base.blueprints]
+oarepo_runtime_info = oarepo_runtime.info.views:create_wellknown_blueprint
+
 [invenio_celery.tasks]
 oarepo_runtime_datastreams = oarepo_runtime.datastreams

{oarepo-runtime-1.5.6 → oarepo-runtime-1.5.7}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = oarepo-runtime
-version = 1.5.6
+version = 1.5.7
 description = A set of runtime extensions of Invenio repository
 authors = Alzbeta Pokorna
 readme = README.md
@@ -50,6 +50,8 @@ invenio_base.api_apps =
 	oarepo_runtime = oarepo_runtime.ext:OARepoRuntime
 invenio_celery.tasks = 
 	oarepo_runtime_datastreams = oarepo_runtime.datastreams
+invenio_base.blueprints = 
+	oarepo_runtime_info = oarepo_runtime.info.views:create_wellknown_blueprint
 
 [oarepo.i18n]
 languages =