oarepo-runtime 1.5.41__tar.gz → 1.5.43__tar.gz

{oarepo_runtime-1.5.41/oarepo_runtime.egg-info → oarepo_runtime-1.5.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oarepo-runtime
-Version: 1.5.41
+Version: 1.5.43
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/cli/fixtures.py

@@ -2,14 +2,7 @@ import click
 import tqdm
 from flask import current_app
 from flask.cli import with_appcontext
-from flask_login import login_user
-from flask_principal import (
-    Identity,
-    RoleNeed,
-    UserNeed,
-    identity_changed,
-    identity_loaded,
-)
+from flask_principal import Identity, RoleNeed, UserNeed
 from invenio_access.permissions import any_user, authenticated_user, system_identity
 from invenio_accounts.models import User
 

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/ext.py

@@ -39,9 +39,9 @@ class OARepoRuntime(object):
 
         for k in ext_config.OAREPO_PERMISSIONS_PRESETS:
             if k not in app.config["OAREPO_PERMISSIONS_PRESETS"]:
-                app.config["OAREPO_PERMISSIONS_PRESETS"][
-                    k
-                ] = ext_config.OAREPO_PERMISSIONS_PRESETS[k]
+                app.config["OAREPO_PERMISSIONS_PRESETS"][k] = (
+                    ext_config.OAREPO_PERMISSIONS_PRESETS[k]
+                )
 
         for k in dir(ext_config):
             if k == "DEFAULT_DATASTREAMS_EXCLUDES":

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/records/systemfields/icu.py

@@ -146,6 +146,10 @@ class ICUSearchField(ICUField):
             "tokenizer": "standard",
             "filter": ["stemming_filter_en"],
         },
+        "ascii_folding_analyzer": {
+            "tokenizer": "standard",
+            "filter": ["ascii_folding_filter"],
+        },
     }
 
     default_stemming_filters = {
@@ -159,6 +163,7 @@ class ICUSearchField(ICUField):
             "name": "english",
             "language": "english",
         },
+        "ascii_folding_filter": {"type": "asciifolding", "preserve_original": True},
     }
 
     def __init__(self, source_field, key=None):
@@ -170,11 +175,24 @@ class ICUSearchField(ICUField):
             self.attr_name: {
                 "type": "object",
                 "properties": {
+                    # normal stemming
                     lang: setting.get(
                         "search",
                         {
                             "type": "text",
-                            "analyzer": f"stemming_analyzer_{lang}",
+                            "boost": 1,
+                            "fields": {
+                                "stemmed": {
+                                    "type": "text",
+                                    "analyzer": f"stemming_analyzer_{lang}",
+                                    "boost": 0.5,
+                                },
+                                "ascii_folded": {
+                                    "type": "text",
+                                    "analyzer": "ascii_folding_analyzer",
+                                    "boost": 0.3,
+                                },
+                            },
                         },
                     )
                     for lang, setting in self.languages.items()

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/records/systemfields/owner.py

@@ -7,8 +7,6 @@
 # details.
 
 """Communities system field."""
-import functools
-import inspect
 
 from invenio_records.systemfields import SystemField
 
@@ -82,12 +80,7 @@ class OwnersField(MappingSystemFieldMixin, SystemField):
         return {
             self.attr_name: {
                 "type": "object",
-                "properties": {
-                    "user": {
-                        "type": "keyword",
-                        "ignore_above": 256
-                    }
-                }
+                "properties": {"user": {"type": "keyword", "ignore_above": 256}},
             },
         }
 

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/records/systemfields/selectors.py

@@ -1,12 +1,12 @@
-from typing import List
+from typing import List, Any
 
 
 class Selector:
-    def select(self, record):
+    def select(self, record) -> List[Any]:
         return []
 
 
-class PathSelector:
+class PathSelector(Selector):
     def __init__(self, *paths):
         self.paths = [x.split(".") for x in paths]
 

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/records/systemfields/synthetic.py

@@ -2,6 +2,7 @@ import logging
 
 from invenio_records.systemfields import SystemField
 
+from . import Selector
 from .mapping import MappingSystemFieldMixin
 
 log = logging.getLogger(__name__)
@@ -69,7 +70,7 @@ class SyntheticSystemField(MappingSystemFieldMixin, SystemField):
            ```
     """
 
-    def __init__(self, selector=None, filter=None, map=None, key=None, **kwargs):
+    def __init__(self, selector: Selector=None, filter=None, map=None, key=None, **kwargs):
         self.selector = selector
         self.map = map
         self.filter = filter

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/services/config/service.py

@@ -4,7 +4,6 @@ from typing import List, Type
 
 from flask import current_app
 from flask_principal import RoleNeed
-from invenio_accounts.models import User
 from invenio_records_permissions import BasePermissionPolicy
 from invenio_records_permissions.generators import Generator
 from invenio_search.engine import dsl

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/services/custom_fields/mappings.py

@@ -19,7 +19,7 @@ from oarepo_runtime.records.systemfields.mapping import MappingSystemFieldMixin
 class Mapping(InvenioMapping):
     @classmethod
     def properties_for_fields(
-            cls, given_fields_names, available_fields, field_name="custom_fields"
+        cls, given_fields_names, available_fields, field_name="custom_fields"
     ):
         """Prepare search mapping properties for each field."""
 
@@ -34,7 +34,7 @@ class Mapping(InvenioMapping):
 
     @classmethod
     def settings_for_fields(
-            cls, given_fields_names, available_fields, field_name="custom_fields"
+        cls, given_fields_names, available_fields, field_name="custom_fields"
     ):
         """Prepare mapping settings for each field."""
 
@@ -87,9 +87,7 @@ def prepare_cf_index(record_class, config, path=[]):
         dynamic_templates = fld.dynamic_templates
 
         for pth in reversed(path):
-            mapping = {
-                pth: mapping
-            }
+            mapping = {pth: mapping}
 
         # upload mapping
         try:
@@ -124,43 +122,24 @@ def prepare_parent_mapping(parent_class, config):
             "properties": {
                 "created": {
                     "type": "date",
-                    "format": "strict_date_time||strict_date_time_no_millis||basic_date_time||basic_date_time_no_millis||basic_date||strict_date||strict_date_hour_minute_second||strict_date_hour_minute_second_fraction"
-                },
-                "id": {
-                    "type": "keyword",
-                    "ignore_above": 1024
+                    "format": "strict_date_time||strict_date_time_no_millis||basic_date_time||basic_date_time_no_millis||basic_date||strict_date||strict_date_hour_minute_second||strict_date_hour_minute_second_fraction",
                 },
+                "id": {"type": "keyword", "ignore_above": 1024},
                 "pid": {
                     "properties": {
-                        "obj_type": {
-                            "type": "keyword",
-                            "ignore_above": 1024
-                        },
-                        "pid_type": {
-                            "type": "keyword",
-                            "ignore_above": 1024
-                        },
-                        "pk": {
-                            "type": "long"
-                        },
-                        "status": {
-                            "type": "keyword",
-                            "ignore_above": 1024
-                        }
+                        "obj_type": {"type": "keyword", "ignore_above": 1024},
+                        "pid_type": {"type": "keyword", "ignore_above": 1024},
+                        "pk": {"type": "long"},
+                        "status": {"type": "keyword", "ignore_above": 1024},
                     }
                 },
                 "updated": {
                     "type": "date",
-                    "format": "strict_date_time||strict_date_time_no_millis||basic_date_time||basic_date_time_no_millis||basic_date||strict_date||strict_date_hour_minute_second||strict_date_hour_minute_second_fraction"
-                },
-                "uuid": {
-                    "type": "keyword",
-                    "ignore_above": 1024
+                    "format": "strict_date_time||strict_date_time_no_millis||basic_date_time||basic_date_time_no_millis||basic_date||strict_date||strict_date_hour_minute_second||strict_date_hour_minute_second_fraction",
                 },
-                "version_id": {
-                    "type": "long"
-                }
-            }
+                "uuid": {"type": "keyword", "ignore_above": 1024},
+                "version_id": {"type": "long"},
+            },
         }
     }
 
@@ -204,6 +183,6 @@ def update_index(record_index, settings, mapping, dynamic_templates=None):
 
 def get_mapping_fields(record_class) -> Iterable[MappingSystemFieldMixin]:
     for cfg_name, cfg_value in inspect.getmembers(
-            record_class, lambda x: isinstance(x, MappingSystemFieldMixin)
+        record_class, lambda x: isinstance(x, MappingSystemFieldMixin)
     ):
         yield cfg_value

oarepo_runtime-1.5.43/oarepo_runtime/services/facets/__init__.py

@@ -0,0 +1,32 @@
+from .base import LabelledValuesTermsFacet
+from .date import (
+    AutoDateHistogramFacet,
+    DateFacet,
+    DateIntervalFacet,
+    DateTimeFacet,
+    EDTFIntervalFacet,
+    TimeFacet,
+)
+from .enum import EnumTermsFacet
+from .facet_groups_names import facet_groups_names
+from .max_facet import MaxFacet
+from .nested_facet import NestedLabeledFacet
+from .params import FilteredFacetsParam, GroupedFacetsParam
+from .year_histogram import YearAutoHistogramFacet
+
+__all__ = [
+    "LabelledValuesTermsFacet",
+    "DateFacet",
+    "TimeFacet",
+    "DateTimeFacet",
+    "AutoDateHistogramFacet",
+    "EDTFIntervalFacet",
+    "DateIntervalFacet",
+    "EnumTermsFacet",
+    "facet_groups_names",
+    "MaxFacet",
+    "NestedLabeledFacet",
+    "GroupedFacetsParam",
+    "FilteredFacetsParam",
+    "YearAutoHistogramFacet",
+]

oarepo_runtime-1.5.43/oarepo_runtime/services/facets/year_histogram.py

@@ -0,0 +1,200 @@
+import re
+from typing import Dict, List
+
+from invenio_records_resources.services.records.facets.facets import LabelledFacetMixin
+from invenio_search.engine import dsl
+
+
+class YearAutoHistogramFacet(LabelledFacetMixin, dsl.Facet):
+    """Histogram facet.
+
+        .. code-block:: python
+
+            facets = {
+                'year': IntegerHistogramFacet(
+                    field='year',
+                    label=_('Year'),
+                    size=1000000
+                )
+            }
+
+            Usage in the oarepo model together with SyntheticSystemField::
+    record:
+      record:
+        imports:
+          - import: oarepo_runtime.records.systemfields.SyntheticSystemField
+          - import: oarepo_runtime.records.systemfields.PathSelector
+        fields:
+          year: |
+            SyntheticSystemField(
+                selector=PathSelector("metadata.date"),
+                key="year",
+                filter=lambda x: len(x) >= 4,
+                map=lambda x: x[:4]
+            )
+      properties:
+        year:
+          facets:
+            facet-class: oarepo_runtime.services.facets.year_histogram.YearAutoHistogramFacet
+          type: edtf
+    """
+
+    agg_type = "auto_date_histogram"
+
+    def __init__(self, **kwargs):
+        self._min_doc_count = kwargs.pop("min_doc_count", 0)
+        buckets = kwargs.pop("buckets", 20)
+        # TODO: the minimum interval should be year, but opensearch does not support it yet
+        super().__init__(
+            **kwargs, buckets=buckets, format="yyyy", minimum_interval="month"
+        )
+
+    def get_value_filter(self, filter_value):
+        if "/" in filter_value:
+            start, end = filter_value.split("/")
+            return dsl.query.Range(
+                _expand__to_dot=False,
+                **{
+                    self._params["field"]: {
+                        "gte": f"{start}-01-01",
+                        "lte": f"{end}-12-31",
+                    }
+                },
+            )
+        return dsl.query.Term(
+            _expand__to_dot=False,
+            **{
+                self._params["field"]: {
+                    "gte": f"{filter_value}-01-01",
+                    "lte": f"{filter_value}-12-31",
+                }
+            },
+        )
+
+    def add_filter(self, filter_values):
+        ret = super().add_filter(filter_values)
+        return ret
+
+    def get_labelled_values(self, data, filter_values):
+        """Get a labelled version of a bucket."""
+
+        # fix for opensearch bug
+        data = self.fix_yearly_interval(data)
+
+        interval = data["interval"]
+
+        interval_in_years = int(re.sub(r"\D", "", interval))
+
+        buckets = data["buckets"]
+
+        for bucket in buckets:
+            bucket["interval"] = interval_in_years
+
+        if self._min_doc_count > 0:
+            buckets = self._merge_small_buckets(buckets)
+
+        out_buckets = []
+        for i, bucket in enumerate(buckets):
+            value = int(bucket["key_as_string"].split("-")[0])
+
+            out_buckets.append(
+                {
+                    **bucket,
+                    "interval": f"{bucket['interval']}y",
+                    "start": str(value),
+                }
+            )
+            if i > 0:
+                out_buckets[i - 1]["end"] = str(value - 1)
+
+        if out_buckets:
+            out_buckets[-1]["end"] = str(
+                int(out_buckets[-1]["start"]) + interval_in_years - 1
+            )
+
+        return {
+            "buckets": out_buckets,
+            "label": str(self._label),
+            "interval": interval,
+        }
+
+    def merge_buckets(self, buckets):
+        merged = {}
+
+        for bucket in buckets:
+            key = bucket["key_as_string"]
+            if key not in merged:
+                merged[key] = {
+                    "key_as_string": key,
+                    "key": bucket["key"],
+                    "doc_count": 0,
+                }
+
+            merged[key]["doc_count"] += bucket["doc_count"]
+
+        result = list(merged.values())
+        return result
+
+    def fix_yearly_interval(self, data) -> Dict:
+        """
+        Currently opensearch has a bug that does not allow to set minimum_interval to year.
+        This function will fix the interval to be yearly if the minimum_interval is has lower value.
+        """
+        data = data.to_dict()
+
+        interval = data["interval"]
+
+        if interval.endswith("y"):
+            # no need to fix the interval, as it is in years
+            return data
+
+        # make sure it is in years
+        data["interval"] = "1y"
+
+        buckets = data["buckets"]
+        data["buckets"] = out_buckets = []
+
+        by_year = {}
+
+        # there might be several buckets returned with the same year - merge them
+        for bucket in buckets:
+            key = bucket["key_as_string"]
+            if key not in by_year:
+                by_year[key] = {
+                    "key_as_string": key,
+                    "key": bucket["key"],
+                    "doc_count": 0,
+                }
+                out_buckets.append(by_year[key])
+
+            by_year[key]["doc_count"] += bucket["doc_count"]
+
+        return data
+
+    def _merge_small_buckets(self, buckets: List[Dict]):
+        """
+        Merges small buckets into the previous bucket. If the small bucket is the first one,
+        merge it with subsequent buckets until the first non-small bucket is found.
+        """
+        ret = []
+        initial_small_buckets = 0
+        initial_small_interval = 0
+        for bucket in buckets:
+            if bucket["doc_count"] < self._min_doc_count:
+                if ret:
+                    ret[-1]["doc_count"] += bucket["doc_count"]
+                    ret[-1]["interval"] += bucket["interval"]
+                else:
+                    initial_small_buckets += bucket["doc_count"]
+                    initial_small_interval += bucket["interval"]
+            else:
+                ret.append(bucket)
+
+        if ret and initial_small_buckets:
+            doc_count = ret[0]["doc_count"] + initial_small_buckets
+            interval = ret[0]["interval"] + initial_small_interval
+            ret[0] = buckets[0]
+            ret[0]["doc_count"] = doc_count
+            ret[0]["interval"] = interval
+
+        return ret

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/services/schema/i18n.py

@@ -1,4 +1,3 @@
-import importlib
 from functools import lru_cache
 
 import langcodes

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/services/schema/i18n_ui.py

@@ -2,7 +2,6 @@ from functools import lru_cache
 
 from invenio_base.utils import obj_or_import_string
 from marshmallow import Schema, fields
-from marshmallow_utils.fields import SanitizedHTML
 
 
 @lru_cache

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime/services/search.py

@@ -14,7 +14,6 @@ from invenio_records_resources.services.records.params import (
 from invenio_records_resources.services.records.queryparser import SuggestQueryParser
 from invenio_search.engine import dsl
 
-
 # TODO: integrate this to invenio_records_resources.services.records and remove SearchOptions class
 from oarepo_runtime.i18n import lazy_gettext as _
 from oarepo_runtime.records.systemfields.icu import ICUSuggestField

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43/oarepo_runtime.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oarepo-runtime
-Version: 1.5.41
+Version: 1.5.43
 Summary: A set of runtime extensions of Invenio repository
 Description-Content-Type: text/markdown
 License-File: LICENSE

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/oarepo_runtime.egg-info/SOURCES.txt

@@ -100,6 +100,7 @@ oarepo_runtime/services/facets/facet_groups_names.py
 oarepo_runtime/services/facets/max_facet.py
 oarepo_runtime/services/facets/nested_facet.py
 oarepo_runtime/services/facets/params.py
+oarepo_runtime/services/facets/year_histogram.py
 oarepo_runtime/services/files/__init__.py
 oarepo_runtime/services/files/components.py
 oarepo_runtime/services/files/service.py

{oarepo_runtime-1.5.41 → oarepo_runtime-1.5.43}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = oarepo-runtime
-version = 1.5.41
+version = 1.5.43
 description = A set of runtime extensions of Invenio repository
 authors = Alzbeta Pokorna
 readme = README.md