oarepo-runtime 1.4.12__tar.gz → 1.4.13__tar.gz

{oarepo-runtime-1.4.12/oarepo_runtime.egg-info → oarepo-runtime-1.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oarepo-runtime
-Version: 1.4.12
+Version: 1.4.13
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -106,71 +106,37 @@ preconfigured permission sets to service config.
 
 To use ICU sort and suggestion custom fields, provide the following configuration
 to `oarepo-model-builder` (or put this stuff to your custom superclasses).
-Please rename those RUNTIME_TEST_XXX to your constants
 
 ```yaml
   record:
     imports:
-      - import: oarepo_runtime.cf.CustomFields
       - import: invenio_records_resources.records.api.Record
         alias: InvenioRecord
-      - import: invenio_records_resources.records.dumpers.CustomFieldsDumperExt
+      - import: oarepo_runtime.records.SystemFieldDumperExt
+      - import: oarepo_runtime.records.icu.ICUSortField
+      - import: oarepo_runtime.records.icu.ICUSuggestField
     extra-code: |-2
           # extra custom fields for testing ICU sorting and suggesting
-          sort = CustomFields(
-              "RUNTIME_TEST_SORT_CF",
-              "sort",
-              clear_none=True,
-              create_if_missing=True,
-          )
-          suggest = CustomFields(
-              "RUNTIME_TEST_SUGGEST_CF",
-              "suggest",
-              clear_none=True,
-              create_if_missing=True,
-          )
+          sort = ICUSortField(source_field="metadata.title")
+          suggest = ICUSuggestField(source_field="metadata.title")
   search-options:
     base-classes:
       - I18nSearchOptions
     imports:
-      - import: oarepo_runtime.services.search.I18nSearchOptions
+      - import: oarepo_runtime.services.icu.I18nSearchOptions
+      - import: oarepo_runtime.services.icu.ICUSuggestParser
+      - import: oarepo_runtime.services.icu.ICUSortOptions
     sort-options-field: extra_sort_options
     extra-code: |-2
-          SORT_CUSTOM_FIELD_NAME = "RUNTIME_TEST_SORT_CF"
-          SUGGEST_CUSTOM_FIELD_NAME = "RUNTIME_TEST_SUGGEST_CF"
+          suggest_parser_cls = ICUSuggestParser("records2")
+          sort_options = ICUSortOptions("records2")
 
   record-dumper:
     extensions:
-      - CustomFieldsDumperExt("RUNTIME_TEST_SORT_CF", "sort")
-      - CustomFieldsDumperExt("RUNTIME_TEST_SUGGEST_CF", "suggest")
+      - SystemFieldDumperExt()
 ```
 
-Then add the constants to your invenio.cfg
-
-```python
-from oarepo_runtime.cf.icu import ICUSortCF, ICUSuggestCF
-
-RUNTIME_TEST_SORT_CF = [
-   ICUSortCF(
-        language = "cs",
-        opensearch_language = "czech",
-        source_field = "metadata.title",
-        # cf_name=None,   # name of generated custom field, default is <language>
-        # country=None,    
-        # variant=None,
-        # sort_option=None, # default is last part of <source_field>
-    )
-]
-RUNTIME_TEST_SUGGEST_CF = [
-   ICUSuggestCF(
-      language="cs", 
-      opensearch_language="czech", 
-      source_field="metadata.title",
-      # cf_name = None    # name of generated custom field, default is <language>
-   )
-]
-```
-
-Then run `invenio oarepo cf init` to initialize custom fields,
-`invenio oarepo index reindex` if you already have data inside the repository
-and from this moment on, `/records?sort=title` 
+Run `invenio oarepo cf init` to initialize custom fields,
+`invenio oarepo index reindex` if you already have data 
+inside the repository and from this moment on, 
+`/records?sort=title` and `/records?suggest=abc` should work

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/README.md

@@ -71,71 +71,37 @@ preconfigured permission sets to service config.
 
 To use ICU sort and suggestion custom fields, provide the following configuration
 to `oarepo-model-builder` (or put this stuff to your custom superclasses).
-Please rename those RUNTIME_TEST_XXX to your constants
 
 ```yaml
   record:
     imports:
-      - import: oarepo_runtime.cf.CustomFields
       - import: invenio_records_resources.records.api.Record
         alias: InvenioRecord
-      - import: invenio_records_resources.records.dumpers.CustomFieldsDumperExt
+      - import: oarepo_runtime.records.SystemFieldDumperExt
+      - import: oarepo_runtime.records.icu.ICUSortField
+      - import: oarepo_runtime.records.icu.ICUSuggestField
     extra-code: |-2
           # extra custom fields for testing ICU sorting and suggesting
-          sort = CustomFields(
-              "RUNTIME_TEST_SORT_CF",
-              "sort",
-              clear_none=True,
-              create_if_missing=True,
-          )
-          suggest = CustomFields(
-              "RUNTIME_TEST_SUGGEST_CF",
-              "suggest",
-              clear_none=True,
-              create_if_missing=True,
-          )
+          sort = ICUSortField(source_field="metadata.title")
+          suggest = ICUSuggestField(source_field="metadata.title")
   search-options:
     base-classes:
       - I18nSearchOptions
     imports:
-      - import: oarepo_runtime.services.search.I18nSearchOptions
+      - import: oarepo_runtime.services.icu.I18nSearchOptions
+      - import: oarepo_runtime.services.icu.ICUSuggestParser
+      - import: oarepo_runtime.services.icu.ICUSortOptions
     sort-options-field: extra_sort_options
     extra-code: |-2
-          SORT_CUSTOM_FIELD_NAME = "RUNTIME_TEST_SORT_CF"
-          SUGGEST_CUSTOM_FIELD_NAME = "RUNTIME_TEST_SUGGEST_CF"
+          suggest_parser_cls = ICUSuggestParser("records2")
+          sort_options = ICUSortOptions("records2")
 
   record-dumper:
     extensions:
-      - CustomFieldsDumperExt("RUNTIME_TEST_SORT_CF", "sort")
-      - CustomFieldsDumperExt("RUNTIME_TEST_SUGGEST_CF", "suggest")
+      - SystemFieldDumperExt()
 ```
 
-Then add the constants to your invenio.cfg
-
-```python
-from oarepo_runtime.cf.icu import ICUSortCF, ICUSuggestCF
-
-RUNTIME_TEST_SORT_CF = [
-   ICUSortCF(
-        language = "cs",
-        opensearch_language = "czech",
-        source_field = "metadata.title",
-        # cf_name=None,   # name of generated custom field, default is <language>
-        # country=None,    
-        # variant=None,
-        # sort_option=None, # default is last part of <source_field>
-    )
-]
-RUNTIME_TEST_SUGGEST_CF = [
-   ICUSuggestCF(
-      language="cs", 
-      opensearch_language="czech", 
-      source_field="metadata.title",
-      # cf_name = None    # name of generated custom field, default is <language>
-   )
-]
-```
-
-Then run `invenio oarepo cf init` to initialize custom fields,
-`invenio oarepo index reindex` if you already have data inside the repository
-and from this moment on, `/records?sort=title` 
+Run `invenio oarepo cf init` to initialize custom fields,
+`invenio oarepo index reindex` if you already have data 
+inside the repository and from this moment on, 
+`/records?sort=title` and `/records?suggest=abc` should work

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/oarepo_runtime/cf/__init__.py

@@ -1,15 +1,45 @@
+from typing import List
+
 from flask import current_app
 from invenio_records.systemfields import DictField, SystemField
+from invenio_records_resources.services.custom_fields import BaseCF
+
+from oarepo_runtime.records import MappingSystemFieldMixin
 
 
-class CustomFieldsMixin:
+class CustomFieldsMixin(MappingSystemFieldMixin):
     def __init__(self, config_key, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.config_key = config_key
 
+    @property
+    def mapping(self):
+        custom_fields: List[BaseCF] = current_app.config[self.config_key]
+        return {cf.name: cf.mapping for cf in custom_fields}
+
+    @property
+    def mapping_settings(self):
+        return {}
+
+    def search_dump(self, data):
+        custom_fields = current_app.config.get(self.config_key, {})
+
+        for cf in custom_fields:
+            cf.dump(data, cf_key=self.key)
+        return data
+
+    def search_load(self, data):
+        custom_fields = current_app.config.get(self.config_key, {})
+
+        for cf in custom_fields:
+            cf.load(data, cf_key=self.key)
+        return data
+
 
 class CustomFields(CustomFieldsMixin, DictField):
-    pass
+    @property
+    def mapping(self):
+        return {self.key: {"type": "object", "properties": super().mapping}}
 
 
 class InlinedCustomFields(CustomFieldsMixin, SystemField):

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/oarepo_runtime/cf/cli.py

@@ -12,15 +12,6 @@ def cf():
 
 
 @cf.command(name="init", help="Prepare custom fields in indices")
-@click.option(
-    "-f",
-    "--field-name",
-    "field_names",
-    type=str,
-    required=False,
-    multiple=True,
-    help="A custom field name to create. If not provided, all custom fields will be created.",
-)
 @with_appcontext
-def init(field_names):
-    prepare_cf_indices(field_names)
+def init():
+    prepare_cf_indices()

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/oarepo_runtime/cf/mappings.py

@@ -2,23 +2,18 @@ import inspect
 from typing import Iterable, List
 
 import click
-from flask import current_app
+import deepmerge
 from invenio_records_resources.proxies import current_service_registry
-from invenio_records_resources.services.custom_fields import BaseCF
 from invenio_records_resources.services.custom_fields.mappings import (
     Mapping as InvenioMapping,
 )
-from invenio_records_resources.services.custom_fields.validate import (
-    validate_custom_fields,
-)
 from invenio_records_resources.services.records.config import RecordServiceConfig
 from invenio_records_resources.services.records.service import RecordService
 from invenio_search import current_search_client
 from invenio_search.engine import dsl, search
 from invenio_search.utils import build_alias_name
 
-from oarepo_runtime.cf import CustomFieldsMixin
-import deepmerge
+from oarepo_runtime.records import MappingSystemFieldMixin
 
 
 class Mapping(InvenioMapping):
@@ -69,34 +64,22 @@ class Mapping(InvenioMapping):
 
 # pieces taken from https://github.com/inveniosoftware/invenio-rdm-records/blob/master/invenio_rdm_records/cli.py
 # as cf initialization is not supported directly in plain invenio
-def prepare_cf_indices(field_names: List[str] = None):
+def prepare_cf_indices():
     service: RecordService
     for service in current_service_registry._services.values():
         config: RecordServiceConfig = service.config
-        prepare_cf_index(config, field_names)
+        prepare_cf_index(config)
 
 
-def prepare_cf_index(config: RecordServiceConfig, field_names: List[str] = None):
+def prepare_cf_index(config: RecordServiceConfig):
     record_class = getattr(config, "record_cls", None)
     if not record_class:
         return
 
-    # try to get custom fields from the record class
-    # validate them
-    for field_name, available_fields in get_custom_fields(record_class):
-        validate_custom_fields(
-            given_fields=field_names, available_fields=available_fields, namespaces=[]
-        )
-
+    for fld in get_mapping_fields(record_class):
         # get mapping
-        properties = Mapping.properties_for_fields(
-            field_names, available_fields, field_name=field_name
-        )
-        settings = Mapping.settings_for_fields(
-            field_names, available_fields, field_name=field_name
-        )
-        if not properties:
-            continue
+        mapping = fld.mapping
+        settings = fld.mapping_settings
 
         # upload mapping
         try:
@@ -106,12 +89,7 @@ def prepare_cf_index(config: RecordServiceConfig, field_names: List[str] = None)
                 ),
                 using=current_search_client,
             )
-            if settings:
-                record_index.close()
-                record_index.put_settings(body=settings)
-                record_index.open()
-            if properties:
-                record_index.put_mapping(body={"properties": properties})
+            update_index(record_index, settings, mapping)
 
             if hasattr(config, "draft_cls"):
                 draft_index = dsl.Index(
@@ -120,15 +98,24 @@ def prepare_cf_index(config: RecordServiceConfig, field_names: List[str] = None)
                     ),
                     using=current_search_client,
                 )
-                draft_index.put_mapping(body={"properties": properties})
+                update_index(draft_index, settings, mapping)
 
         except search.RequestError as e:
             click.secho("An error occurred while creating custom fields.", fg="red")
             click.secho(e.info["error"]["reason"], fg="red")
 
 
-def get_custom_fields(record_class) -> Iterable[List[BaseCF]]:
+def update_index(record_index, settings, mapping):
+    if settings:
+        record_index.close()
+        record_index.put_settings(body=settings)
+        record_index.open()
+    if mapping:
+        record_index.put_mapping(body={"properties": mapping})
+
+
+def get_mapping_fields(record_class) -> Iterable[MappingSystemFieldMixin]:
     for cfg_name, cfg_value in inspect.getmembers(
-        record_class, lambda x: isinstance(x, CustomFieldsMixin)
+        record_class, lambda x: isinstance(x, MappingSystemFieldMixin)
     ):
-        yield cfg_value._key, current_app.config[cfg_value.config_key]
+        yield cfg_value

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/oarepo_runtime/cli/index.py

@@ -1,4 +1,3 @@
-import itertools
 import sys
 
 import click

oarepo-runtime-1.4.13/oarepo_runtime/records/__init__.py

@@ -0,0 +1,29 @@
+import inspect
+
+from invenio_records.dumpers import SearchDumperExt
+
+
+class MappingSystemFieldMixin:
+    @property
+    def mapping(self):
+        return {}
+
+    @property
+    def mapping_settings(self):
+        return {}
+
+
+class SystemFieldDumperExt(SearchDumperExt):
+    def dump(self, record, data):
+        """Dump custom fields."""
+        for cf in inspect.getmembers(
+            record, lambda x: isinstance(x, MappingSystemFieldMixin)
+        ):
+            cf[1].search_dump(data)
+
+    def load(self, data, record_cls):
+        """Load custom fields."""
+        for cf in inspect.getmembers(
+            record_cls, lambda x: isinstance(x, MappingSystemFieldMixin)
+        ):
+            cf[1].search_load(data)

oarepo-runtime-1.4.13/oarepo_runtime/records/icu.py

@@ -0,0 +1,132 @@
+from functools import cached_property
+from typing import Dict
+
+from flask import current_app
+from invenio_records.systemfields import SystemField
+
+from oarepo_runtime.records import MappingSystemFieldMixin
+from oarepo_runtime.relations.lookup import lookup_key
+
+
+class ICUField(SystemField):
+    """
+    A system field that acts as an opensearch "proxy" to another field.
+    It creates a top-level mapping field with the same name and copies
+    content of {another field}.language into {mapping field}.language.
+
+    The language accessor can be modified by overriding get_values method.
+    """
+
+    def __init__(self, *, source_field, key=None):
+        super().__init__(key)
+        self.source_field = source_field
+
+    @cached_property
+    def languages(self) -> Dict[str, Dict]:
+        icu_languages = current_app.config.get("OAREPO_ICU_LANGUAGES", {})
+        if icu_languages:
+            return icu_languages
+
+        primary_language = current_app.config.get("BABEL_DEFAULT_LOCALE", "en")
+        # list of tuples [lang, title], just take lang
+        babel_languages = [x[0] for x in current_app.config.get("I18N_LANGUAGES", [])]
+
+        return {primary_language: {}, **{k: {} for k in babel_languages}}
+
+    def get_values(self, data, language):
+        ret = []
+        for l in lookup_key(data, f"{self.source_field}"):
+            if isinstance(l.value, str):
+                ret.append(l.value)
+            elif isinstance(l.value, dict):
+                val = l.value.get(language)
+                if val:
+                    ret.append(val)
+        return ret
+
+    def search_dump(self, data):
+        ret = {}
+        for lang in self.languages:
+            ret[lang] = self.get_values(data, lang)
+        data[self.attr_name] = ret
+
+    def search_load(self, data):
+        data.pop(self.attr_name, None)
+
+    def __get__(self, instance, owner):
+        return self
+
+
+class ICUSortField(MappingSystemFieldMixin, ICUField):
+    """
+    A field that adds icu sorting field
+    """
+
+    def __init__(self, *, source_field, key=None):
+        super().__init__(source_field=source_field, key=key)
+
+    @property
+    def mapping(self):
+        return {
+            self.attr_name: {
+                "type": "object",
+                "properties": {
+                    lang: {
+                        "type": "icu_collation_keyword",
+                        "index": False,
+                        "language": lang,
+                        **setting.get("collation", {}),
+                    }
+                    for lang, setting in self.languages.items()
+                },
+            },
+        }
+
+
+class ICUSuggestField(MappingSystemFieldMixin, ICUField):
+    """
+    A field that adds icu-aware suggestion field
+    """
+
+    def __init__(self, source_field, key=None):
+        super().__init__(source_field=source_field, key=key)
+
+    @property
+    def mapping(self):
+        return {
+            self.attr_name: {
+                "type": "object",
+                "properties": {
+                    lang: setting.get(
+                        "suggest",
+                        {
+                            "type": "text",
+                            "fields": {
+                                "original": {
+                                    "type": "search_as_you_type",
+                                },
+                                "no_accent": {
+                                    "type": "search_as_you_type",
+                                    "analyzer": "accent_removal_analyzer",
+                                },
+                            },
+                        },
+                    )
+                    for lang, setting in self.languages.items()
+                },
+            },
+        }
+
+    @property
+    def mapping_settings(self):
+        return {
+            "analysis": {
+                "analyzer": {
+                    "accent_removal_analyzer": {
+                        "type": "custom",
+                        "tokenizer": "standard",
+                        "filter": ["lowercase", "asciifolding"],
+                    }
+                }
+            }
+        }

{oarepo-runtime-1.4.12 → oarepo-runtime-1.4.13}/oarepo_runtime/relations/errors.py

@@ -12,5 +12,7 @@ class MultipleInvalidRelationErrors(Exception):
     """
 
     def __init__(self, errors):
-        super().__init__("; ".join([f"{e[0]}: {type(e[1]).__name__}({e[1]})" for e in errors]))
+        super().__init__(
+            "; ".join([f"{e[0]}: {type(e[1]).__name__}({e[1]})" for e in errors])
+        )
         self.errors = errors