oarepo-runtime 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

oarepo_runtime/records/systemfields/__init__.py

@@ -1,12 +1,26 @@
-from .icu import ICUField, ICUSortField, ICUSuggestField
+from .icu import (
+    FulltextIndexField,
+    ICUField,
+    ICUSearchField,
+    ICUSortField,
+    ICUSuggestField,
+)
 from .mapping import MappingSystemFieldMixin, SystemFieldDumperExt
-from .selectors import FirstItemSelector, PathSelector, Selector, FilteredSelector, MultiSelector
+from .selectors import (
+    FilteredSelector,
+    FirstItemSelector,
+    MultiSelector,
+    PathSelector,
+    Selector,
+)
 from .synthetic import SyntheticSystemField
 
 __all__ = (
     "ICUField",
     "ICUSuggestField",
     "ICUSortField",
+    "ICUSearchField",
+    "FulltextIndexField",
     "MappingSystemFieldMixin",
     "SystemFieldDumperExt",
     "SyntheticSystemField",

oarepo_runtime/records/systemfields/icu.py

@@ -37,11 +37,18 @@ class ICUField(MappingSystemFieldMixin, SystemField):
         ret = []
         for l in lookup_key(data, f"{self.source_field}"):
             if isinstance(l.value, str):
+                # take single value as being always the the language provided
                 ret.append(l.value)
             elif isinstance(l.value, dict):
+                # expected to be {"cs": "", "en": ""}
                 val = l.value.get(language)
                 if val:
                     ret.append(val)
+                elif "lang" in l.value:
+                    # for [{"lang": "", "value": ""}, ...] we get each item separately
+                    # that's why we do not iterate over l.value
+                    if l.value["lang"] == language:
+                        ret.append(l.value["value"])
         return ret
 
     def search_dump(self, data, record):
@@ -132,23 +139,24 @@ class ICUSuggestField(ICUField):
         }
 
 
-class ICUSearchField(ICUField):
-    """
-    A field that adds stemming-aware search field
-    """
+class ICUSearchAnalyzerMixin:
 
     default_stemming_analyzers = {
         "stemming_analyzer_cs": {
             "tokenizer": "standard",
-            "filter": ["stemming_filter_cs"],
+            "filter": ["stemming_filter_cs", "lowercase"],
         },
         "stemming_analyzer_en": {
             "tokenizer": "standard",
-            "filter": ["stemming_filter_en"],
+            "filter": ["stemming_filter_en", "lowercase"],
         },
         "ascii_folding_analyzer": {
             "tokenizer": "standard",
-            "filter": ["ascii_folding_filter"],
+            "filter": ["ascii_folding_filter", "lowercase"],
+        },
+        "lowercase_analyzer": {
+            "tokenizer": "standard",
+            "filter": ["lowercase"],
         },
     }
 
@@ -166,6 +174,25 @@ class ICUSearchField(ICUField):
         "ascii_folding_filter": {"type": "asciifolding", "preserve_original": True},
     }
 
+    @property
+    def mapping_settings(self):
+        return {
+            "analysis": {
+                "analyzer": current_app.config.get(
+                    "OAREPO_ICU_SEARCH_ANALYZERS", self.default_stemming_analyzers
+                ),
+                "filter": current_app.config.get(
+                    "OAREPO_ICU_SEARCH_FILTERS", self.default_stemming_filters
+                ),
+            }
+        }
+
+
+class ICUSearchField(ICUSearchAnalyzerMixin, ICUField):
+    """
+    A field that adds stemming-aware search field
+    """
+
     def __init__(self, source_field, key=None):
         super().__init__(source_field=source_field, key=key)
 
@@ -187,6 +214,11 @@ class ICUSearchField(ICUField):
                                     "analyzer": f"stemming_analyzer_{lang}",
                                     "boost": 0.5,
                                 },
+                                "lowercase": {
+                                    "type": "text",
+                                    "boost": 0.8,
+                                    "analyzer": "lowercase_analyzer",
+                                },
                                 "ascii_folded": {
                                     "type": "text",
                                     "analyzer": "ascii_folding_analyzer",
@@ -200,15 +232,59 @@ class ICUSearchField(ICUField):
             },
         }
 
+    def get_values(self, data, language):
+        return super().get_values(data, language=language)
+
+
+class FulltextIndexField(ICUSearchAnalyzerMixin, ICUField):
+    """
+    A system field that makes the field searchable in OpenSearch,
+    regardless if it is indexed/analyzed, embedded in Nested or not.
+
+    It creates a top-level mapping field and copies
+    content of {source_field} into it. It also provides the correct mapping
+    for the field based on the current configuration of the application.
+
+    Unlike the ICU, this field is a single-language and the language should
+    be provided when initializing the field.
+    It defaults to the BABEL_DEFAULT_LOCALE if not provided.
+    """
+
+    def __init__(self, *, source_field, key=None, language=None):
+        super().__init__(source_field=source_field, key=key)
+        self.language = language
+
     @property
-    def mapping_settings(self):
-        return {
-            "analysis": {
-                "analyzer": current_app.config.get(
-                    "OAREPO_ICU_SEARCH_ANALYZERS", self.default_stemming_analyzers
-                ),
-                "filter": current_app.config.get(
-                    "OAREPO_ICU_SEARCH_FILTERS", self.default_stemming_filters
-                ),
+    def mapping(self):
+        language = self.language or current_app.config.get("BABEL_DEFAULT_LOCALE", "en")
+        mapping_settings = self.languages.get(language, None)
+        if mapping_settings:
+            mapping_settings = mapping_settings.get("search")
+        if not mapping_settings:
+            mapping_settings = {
+                "type": "text",
+                "boost": 1,
+                "fields": {
+                    "stemmed": {
+                        "type": "text",
+                        "analyzer": f"stemming_analyzer_{language}",
+                        "boost": 0.5,
+                    },
+                    "ascii_folded": {
+                        "type": "text",
+                        "analyzer": "ascii_folding_analyzer",
+                        "boost": 0.3,
+                    },
+                },
             }
-        }
+
+        return {self.attr_name: mapping_settings}
+
+    def search_dump(self, data, record):
+        """Dump custom field."""
+        data[self.attr_name] = self.get_values(data, language=self.language)
+
+    @classmethod
+    def search_load(cls, data, record_cls):
+        """Load custom field."""
+        data.pop(cls.attr_name, None)

oarepo_runtime/services/custom_fields/mappings.py

@@ -9,17 +9,16 @@ from invenio_records_resources.services.custom_fields.mappings import (
 )
 from invenio_records_resources.services.records.config import RecordServiceConfig
 from invenio_records_resources.services.records.service import RecordService
-from invenio_search import current_search_client
-from invenio_search.engine import dsl, search
-from invenio_search.utils import build_alias_name
-
+from invenio_search.engine import search
 from deepmerge import always_merger
 from oarepo_runtime.records.systemfields.mapping import MappingSystemFieldMixin
 import json
-import os
 
 from pathlib import Path
 
+from oarepo_runtime.utils.index import prefixed_index
+
+
 class Mapping(InvenioMapping):
     @classmethod
     def properties_for_fields(
@@ -95,21 +94,11 @@ def prepare_cf_index(record_class, config, path=[]):
 
         # upload mapping
         try:
-            record_index = dsl.Index(
-                build_alias_name(
-                    config.record_cls.index._name,
-                ),
-                using=current_search_client,
-            )
+            record_index = prefixed_index(config.record_cls.index)
             update_index(record_index, settings, mapping)
 
             if hasattr(config, "draft_cls"):
-                draft_index = dsl.Index(
-                    build_alias_name(
-                        config.draft_cls.index._name,
-                    ),
-                    using=current_search_client,
-                )
+                draft_index = prefixed_index(config.draft_cls.index)
                 update_index(draft_index, settings, mapping, dynamic_templates)
 
         except search.RequestError as e:
@@ -166,22 +155,12 @@ def prepare_parent_mapping(parent_class, config):
     })
     # upload mapping
     try:
-        record_index = dsl.Index(
-            build_alias_name(
-                config.record_cls.index._name,
-            ),
-            using=current_search_client,
-        )
+        record_index = prefixed_index(config.record_cls.index)
         update_index(record_index, {}, parent_mapping_merged)
 
         if hasattr(config, "draft_cls"):
-            draft_index = dsl.Index(
-                build_alias_name(
-                    config.draft_cls.index._name,
-                ),
-                using=current_search_client,
-            )
-            update_index(record_index, {}, parent_mapping_merged)
+            draft_index = prefixed_index(config.draft_cls.index) # draft index isn't used; this was a bug a suppose
+            update_index(draft_index, {}, parent_mapping_merged)
 
     except search.RequestError as e:
         click.secho("An error occurred while creating parent mapping.", fg="red")

oarepo_runtime/utils/index.py

@@ -0,0 +1,11 @@
+from invenio_search import current_search_client
+from invenio_search.utils import build_alias_name
+from invenio_search.engine import dsl
+
+def prefixed_index(index):
+    return dsl.Index(
+        build_alias_name(
+            index._name,
+        ),
+        using = current_search_client
+    )

{oarepo_runtime-1.7.1.dist-info → oarepo_runtime-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oarepo-runtime
-Version: 1.7.1
+Version: 1.8.0
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE

{oarepo_runtime-1.7.1.dist-info → oarepo_runtime-1.8.0.dist-info}/RECORD

@@ -70,10 +70,10 @@ oarepo_runtime/records/relations/base.py,sha256=ESTwj0-eT8HRTJ8QcE5fmqzjOjBFHpQq
 oarepo_runtime/records/relations/internal.py,sha256=OTp8iJqyl80sWDk0Q0AK42l6UsxZDABspVU_GwWza9o,1556
 oarepo_runtime/records/relations/lookup.py,sha256=wi3jPfOedazOmhOMrgu50PUETc1jfSdpmjK0wvOFsEM,848
 oarepo_runtime/records/relations/pid_relation.py,sha256=eojw5uIo5zXmJGge_bj6Wj2njCRY5S4o4B_h_HFyaDY,3901
-oarepo_runtime/records/systemfields/__init__.py,sha256=LL1R64RUakA_4r0IkTq9MtwqD5eV-AQaj5u96zkWa74,533
+oarepo_runtime/records/systemfields/__init__.py,sha256=fdOvVL7L-Q4pcNHPcixYw52ydwEd9mBJVVvZT2-6ON4,667
 oarepo_runtime/records/systemfields/featured_file.py,sha256=MbSaYR130_o5S9gEOblnChq-PVK4xGPGpSCrzwG3cwc,1720
 oarepo_runtime/records/systemfields/has_draftcheck.py,sha256=4JkMEefPLpqtPtlTgK3UT0KzTRgyw5_Qtkss2qcz5xk,1643
-oarepo_runtime/records/systemfields/icu.py,sha256=sSGAgi5WhsAY4cCBL7-7nMpvHAuctpW8Y8vRExHQUfk,6738
+oarepo_runtime/records/systemfields/icu.py,sha256=id4yv80DG-8XWMaSeegpoF7JWciXTHq69U2Xqo_4lW8,9700
 oarepo_runtime/records/systemfields/mapping.py,sha256=tXOK_jkdY1pOUO7_VfChfDNB8UTi21GUXaidpugTnO8,1017
 oarepo_runtime/records/systemfields/owner.py,sha256=dYRVBinniW7ECHuSnTAjeN6x1KhhJtNR9vxmD1KswMs,3805
 oarepo_runtime/records/systemfields/record_status.py,sha256=U3kem4-JkNsT17e0iAl3HIAZ2MvO5lY_0U757aZvTKE,935
@@ -98,7 +98,7 @@ oarepo_runtime/services/config/link_conditions.py,sha256=5hDEtCaQt1tzjb9gGdeimsm
 oarepo_runtime/services/config/permissions_presets.py,sha256=b1wm3JjL8KgkaH7VMkNfdMk5fXtf3X87rB7gkvaNxns,7369
 oarepo_runtime/services/config/service.py,sha256=KS1eKXjRzusD_UAKanzKezua2lmo9vObwqd63dky1ik,4593
 oarepo_runtime/services/custom_fields/__init__.py,sha256=_gqMcA_I3rdEZcBtCuDjO4wdVCqFML5NzaccuPx5a3o,2565
-oarepo_runtime/services/custom_fields/mappings.py,sha256=3CIvjWzVRO3fZ2r-3taK1rwuLH9wj5Lguo-GvsbLBf4,7515
+oarepo_runtime/services/custom_fields/mappings.py,sha256=U43noOuANohgoZt7YoAT3B6NgDCLKIRVKigPHzsreQU,6970
 oarepo_runtime/services/entity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oarepo_runtime/services/entity/config.py,sha256=1jfdPrxSbMuKj7eOUNKRWTCPbBPyRV6MrWE4Vgf9rX0,399
 oarepo_runtime/services/entity/schema.py,sha256=8TBpUFRITaBO7qCMz36cly1Hj4I1nLa9PeSAfWSa2YM,157
@@ -148,14 +148,15 @@ oarepo_runtime/translations/en/LC_MESSAGES/messages.mo,sha256=tq5pTCpH7cuzdwrQlp
 oarepo_runtime/translations/en/LC_MESSAGES/messages.po,sha256=7-5S3iINOSFSI5gVdRvT5S2rbGScrheJiUJQH_fqXmY,1914
 oarepo_runtime/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 oarepo_runtime/utils/functools.py,sha256=gKS9YZtlIYcDvdNA9cmYO00yjiXBYV1jg8VpcRUyQyg,1324
+oarepo_runtime/utils/index.py,sha256=ArrUUXB-KowUcUksRKqcFpmqct4bn9alO1zd_kX2tmU,292
 oarepo_runtime/utils/path.py,sha256=V1NVyk3m12_YLbj7QHYvUpE1wScO78bYsX1LOLeXDkI,3108
-oarepo_runtime-1.7.1.dist-info/licenses/LICENSE,sha256=h2uWz0OaB3EN-J1ImdGJZzc7yvfQjvHVYdUhQ-H7ypY,1064
+oarepo_runtime-1.8.0.dist-info/licenses/LICENSE,sha256=h2uWz0OaB3EN-J1ImdGJZzc7yvfQjvHVYdUhQ-H7ypY,1064
 tests/marshmallow_to_json/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/marshmallow_to_json/test_datacite_ui_schema.py,sha256=82iLj8nW45lZOUewpWbLX3mpSkpa9lxo-vK-Qtv_1bU,48552
 tests/marshmallow_to_json/test_simple_schema.py,sha256=izZN9p0v6kovtSZ6AdxBYmK_c6ZOti2_z_wPT_zXIr0,1500
 tests/pkg_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oarepo_runtime-1.7.1.dist-info/METADATA,sha256=rvL7MLlWJ21OM6vVVDks-Ga6iqUJiqmqEZirtr1OfbA,4788
-oarepo_runtime-1.7.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-oarepo_runtime-1.7.1.dist-info/entry_points.txt,sha256=k7O5LZUOGsVeSpB7ulU0txBUNp1CVQG7Q7TJIVTPbzU,491
-oarepo_runtime-1.7.1.dist-info/top_level.txt,sha256=bHhlkT1_RQC4IkfTQCqA3iN4KCB6cSFQlsXpQMSP-bE,21
-oarepo_runtime-1.7.1.dist-info/RECORD,,
+oarepo_runtime-1.8.0.dist-info/METADATA,sha256=2RIMYE7SkyjV-UjfAVIT-CBXIOk8ImbsxDrIweni-Mw,4788
+oarepo_runtime-1.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+oarepo_runtime-1.8.0.dist-info/entry_points.txt,sha256=k7O5LZUOGsVeSpB7ulU0txBUNp1CVQG7Q7TJIVTPbzU,491
+oarepo_runtime-1.8.0.dist-info/top_level.txt,sha256=bHhlkT1_RQC4IkfTQCqA3iN4KCB6cSFQlsXpQMSP-bE,21
+oarepo_runtime-1.8.0.dist-info/RECORD,,