oarepo-runtime 1.10.2__py3-none-any.whl → 2.0.0.dev3__py3-none-any.whl

oarepo_runtime/records/systemfields/icu.py

@@ -1,371 +0,0 @@
-from abc import abstractmethod, abstractproperty
-from functools import cached_property
-from typing import Dict
-
-from flask import current_app
-from invenio_records.systemfields import SystemField
-
-from oarepo_runtime.records.relations.lookup import lookup_key
-from oarepo_runtime.records.systemfields.mapping import MappingSystemFieldMixin
-
-
-class ICUBase(MappingSystemFieldMixin, SystemField):
-    """
-    Base class for ICU system fields.
-    It provides the basic functionality for ICU fields, such as
-    getting the attribute name and handling the key.
-    """
-
-    def __init__(self, source_field=None, key=None):
-        super().__init__(key=key)
-        self._attr_name = key or self.__class__.__name__.lower()
-        self.source_field = source_field
-
-    @cached_property
-    def languages(self) -> Dict[str, Dict]:
-        icu_languages = current_app.config.get("OAREPO_ICU_LANGUAGES", {})
-        if icu_languages:
-            return icu_languages
-
-        primary_language = current_app.config.get("BABEL_DEFAULT_LOCALE", "en")
-        # list of tuples [lang, title], just take lang
-        babel_languages = [x[0] for x in current_app.config.get("I18N_LANGUAGES", [])]
-
-        return {primary_language: {}, **{k: {} for k in babel_languages}}
-
-    def get_values(self, data, language):
-        ret = []
-        for l in lookup_key(data, f"{self.source_field}"):
-            if isinstance(l.value, str):
-                # take single value as being always the the language provided
-                ret.append(l.value)
-            elif isinstance(l.value, dict):
-                # expected to be {"cs": "", "en": ""}
-                val = l.value.get(language)
-                if val:
-                    ret.append(val)
-                elif "lang" in l.value:
-                    # for [{"lang": "", "value": ""}, ...] we get each item separately
-                    # that's why we do not iterate over l.value
-                    if l.value["lang"] == language:
-                        ret.append(l.value["value"])
-        return ret
-
-    @abstractproperty
-    def mapping(self):
-        """
-        The mapping for the field. It should return a dictionary with the
-        mapping for the field, based on the current configuration of the application.
-        """
-        raise NotImplementedError("Subclasses must implement the mapping property.")
-
-    @abstractmethod
-    def search_dump(self, data, record):
-        """
-        Dump custom field. This method should be implemented by subclasses
-        to provide the functionality for dumping the field data into the
-        OpenSearch data structure.
-        """
-        raise NotImplementedError("Subclasses must implement the search_dump method.")
-
-    def search_load(self, data, record_cls):
-        """
-        Just remove the field from the data on load.
-        """
-        data.pop(self.attr_name, None)
-
-    def __get__(self, instance, owner):
-        return self
-
-
-class ICUField(ICUBase):
-    """
-    A system field that acts as an opensearch "proxy" to another field.
-    It creates a top-level mapping field with the same name and copies
-    content of {another field}.language into {mapping field}.language.
-
-    The language accessor can be modified by overriding get_values method.
-    """
-
-    def __init__(self, *, source_field, key=None):
-        super().__init__(source_field=source_field, key=key)
-
-    def search_dump(self, data, record):
-        ret = {}
-        for lang in self.languages:
-            r = self.get_values(data, lang)
-            if r:
-                # if the language is not empty, add it to the result
-                # otherwise do not add it at all to safe transport
-                ret[lang] = r
-        if ret:
-            data[self.attr_name] = ret
-
-
-class ICUSortField(ICUField):
-    """
-    A field that adds icu sorting field
-    """
-
-    def __init__(self, *, source_field, key=None):
-        super().__init__(source_field=source_field, key=key)
-
-    @property
-    def mapping(self):
-        return {
-            self.attr_name: {
-                "type": "object",
-                "properties": {
-                    lang: {
-                        "type": "icu_collation_keyword",
-                        "index": False,
-                        "language": lang,
-                        **setting.get("collation", {}),
-                    }
-                    for lang, setting in self.languages.items()
-                },
-            },
-        }
-
-
-class ICUSuggestField(ICUField):
-    """
-    A field that adds icu-aware suggestion field
-    """
-
-    def __init__(self, source_field, key=None):
-        super().__init__(source_field=source_field, key=key)
-
-    @property
-    def mapping(self):
-        return {
-            self.attr_name: {
-                "type": "object",
-                "properties": {
-                    lang: setting.get(
-                        "suggest",
-                        {
-                            "type": "text",
-                            "fields": {
-                                "original": {
-                                    "type": "search_as_you_type",
-                                },
-                                "no_accent": {
-                                    "type": "search_as_you_type",
-                                    "analyzer": "accent_removal_analyzer",
-                                },
-                            },
-                        },
-                    )
-                    for lang, setting in self.languages.items()
-                },
-            },
-        }
-
-    @property
-    def mapping_settings(self):
-        return {
-            "analysis": {
-                "analyzer": {
-                    "accent_removal_analyzer": {
-                        "type": "custom",
-                        "tokenizer": "standard",
-                        "filter": ["lowercase", "asciifolding"],
-                    }
-                }
-            }
-        }
-
-
-class ICUSearchAnalyzerMixin:
-
-    default_stemming_analyzers = {
-        "stemming_analyzer_cs": {
-            "tokenizer": "standard",
-            "filter": ["stemming_filter_cs", "lowercase"],
-        },
-        "stemming_analyzer_en": {
-            "tokenizer": "standard",
-            "filter": ["stemming_filter_en", "lowercase"],
-        },
-        "ascii_folding_analyzer": {
-            "tokenizer": "standard",
-            "filter": ["ascii_folding_filter", "lowercase"],
-        },
-        "lowercase_analyzer": {
-            "tokenizer": "standard",
-            "filter": ["lowercase"],
-        },
-    }
-
-    default_stemming_filters = {
-        "stemming_filter_cs": {
-            "type": "stemmer",
-            "name": "czech",
-            "language": "czech",
-        },
-        "stemming_filter_en": {
-            "type": "stemmer",
-            "name": "english",
-            "language": "english",
-        },
-        "ascii_folding_filter": {"type": "asciifolding", "preserve_original": True},
-    }
-
-    @property
-    def mapping_settings(self):
-        return {
-            "analysis": {
-                "analyzer": current_app.config.get(
-                    "OAREPO_ICU_SEARCH_ANALYZERS", self.default_stemming_analyzers
-                ),
-                "filter": current_app.config.get(
-                    "OAREPO_ICU_SEARCH_FILTERS", self.default_stemming_filters
-                ),
-            }
-        }
-
-
-class ICUSearchField(ICUSearchAnalyzerMixin, ICUField):
-    """
-    A field that adds stemming-aware search field for multilingual data (
-        e.g. data that contains {"cs": "...", "en": "..."}
-        or [{"lang": "cs", "value": "..."}, ...]
-    )
-    """
-
-    def __init__(self, source_field, key=None, boost=1):
-        super().__init__(source_field=source_field, key=key)
-        self.boost = boost
-
-    @property
-    def mapping(self):
-        return {
-            self.attr_name: {
-                "type": "object",
-                "properties": {
-                    # normal stemming
-                    lang: setting.get(
-                        "search",
-                        {
-                            "type": "text",
-                            "boost": 1 * self.boost,
-                            "fields": {
-                                "stemmed": {
-                                    "type": "text",
-                                    "analyzer": f"stemming_analyzer_{lang}",
-                                    "boost": 0.5 * self.boost,
-                                },
-                                "lowercase": {
-                                    "type": "text",
-                                    "boost": 0.8 * self.boost,
-                                    "analyzer": "lowercase_analyzer",
-                                },
-                                "ascii_folded": {
-                                    "type": "text",
-                                    "analyzer": "ascii_folding_analyzer",
-                                    "boost": 0.3 * self.boost,
-                                },
-                            },
-                        },
-                    )
-                    for lang, setting in self.languages.items()
-                },
-            },
-        }
-
-    def get_values(self, data, language):
-        return super().get_values(data, language=language)
-
-
-class SingleLanguageSearchField(ICUSearchAnalyzerMixin, ICUBase):
-    """
-    A base class for single-language search fields - that is, data contain a text
-    value in a pre-defined, single language.
-    """
-
-    def __init__(self, *, source_field, key=None, language=None, boost=1):
-        super().__init__(source_field=source_field, key=key)
-        self.language = language
-        self.boost = boost
-
-    def search_dump(self, data, record):
-        """Dump custom field."""
-        ret = self.get_values(data, language=self.language)
-        if ret:
-            data[self.attr_name] = ret
-
-
-class FulltextIndexField(SingleLanguageSearchField):
-    """
-    A system field that makes the field searchable in OpenSearch,
-    regardless if it is indexed/analyzed, embedded in Nested or not.
-
-    It creates a top-level mapping field and copies
-    content of {source_field} into it. It also provides the correct mapping
-    for the field based on the current configuration of the application.
-
-    Unlike the ICU, this field is a single-language and the language should
-    be provided when initializing the field.
-    It defaults to the BABEL_DEFAULT_LOCALE if not provided.
-    """
-
-    @property
-    def mapping(self):
-        language = self.language or current_app.config.get("BABEL_DEFAULT_LOCALE", "en")
-        mapping_settings = self.languages.get(language, None)
-        if mapping_settings:
-            mapping_settings = mapping_settings.get("search")
-        if not mapping_settings:
-            mapping_settings = {
-                "type": "text",
-                "boost": 1 * self.boost,
-                "fields": {
-                    "stemmed": {
-                        "type": "text",
-                        "analyzer": f"stemming_analyzer_{language}",
-                        "boost": 0.5 * self.boost,
-                    },
-                    "lowercase": {
-                        "type": "text",
-                        "boost": 0.8 * self.boost,
-                        "analyzer": "lowercase_analyzer",
-                    },
-                    "ascii_folded": {
-                        "type": "text",
-                        "analyzer": "ascii_folding_analyzer",
-                        "boost": 0.3 * self.boost,
-                    },
-                },
-            }
-
-        return {self.attr_name: mapping_settings}
-
-    def search_load(self, data, record_cls):
-        """Load custom field."""
-        data.pop(self.attr_name, None)
-
-
-class TermIndexField(SingleLanguageSearchField):
-    """
-    A system field that makes the field searchable in OpenSearch,
-    regardless if it is indexed/analyzed, embedded in Nested or not.
-
-    It creates a top-level mapping field and copies
-    content of {source_field} into it. It also provides the correct mapping
-    for the field based on the current configuration of the application.
-
-    Unlike the ICU, this field is a single-language and the language should
-    be provided when initializing the field.
-    It defaults to the BABEL_DEFAULT_LOCALE if not provided.
-    """
-
-    @property
-    def mapping(self):
-        mapping_settings = {
-            "type": "keyword",
-            "boost": 1 * self.boost,
-            "ignore_above": 256,
-        }
-
-        return {self.attr_name: mapping_settings}

oarepo_runtime/records/systemfields/owner.py

@@ -1,115 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2021 CERN.
-#
-# Invenio-Communities is free software; you can redistribute it and/or
-# modify it under the terms of the MIT License; see LICENSE file for more
-# details.
-
-"""Communities system field."""
-
-from invenio_records.systemfields import SystemField
-
-from oarepo_runtime.records.owners import OwnerEntityResolverRegistry
-from oarepo_runtime.records.systemfields import MappingSystemFieldMixin
-
-
-class OwnerRelationManager:
-    def __init__(self, record_id, serialized_owners):
-        self._serialized_owners = serialized_owners
-        self._deserialized_owners = None
-
-    # from oarepo_requests utils, dependancy on that would be wrong here, right?
-    # invenio_requests is ok<
-
-    #
-    # API
-    #
-
-    def to_dict(self):
-        if self._serialized_owners is None:
-            deserialized_owners = []
-            for deserialized_owner in self._deserialized_owners or []:
-                serialized_owner = OwnerEntityResolverRegistry.reference_entity(
-                    deserialized_owner
-                )
-                if serialized_owner is None:
-                    raise ValueError(
-                        f"failed serialize owner; owner - {deserialized_owner}"
-                    )
-                deserialized_owners.append(serialized_owner)
-            self._serialized_owners = deserialized_owners
-        return self._serialized_owners
-
-    def _resolve(self):
-        if self._deserialized_owners is None:
-            self._deserialized_owners = set()
-            for ref in self._serialized_owners or []:
-                self._deserialized_owners.add(
-                    OwnerEntityResolverRegistry.resolve_reference(ref)
-                )
-            self._serialized_owners = None
-
-    def add(self, owner):
-        if owner is None:
-            return
-        self._resolve()
-        self._deserialized_owners.add(owner)
-
-    def remove(self, owner):
-        if owner is None:
-            return
-        self._resolve()
-        self._deserialized_owners.remove(owner)
-
-    def __iter__(self):
-        self._resolve()
-        return iter(self._deserialized_owners)
-
-
-class OwnersField(MappingSystemFieldMixin, SystemField):
-    """Communites system field for managing relations to communities."""
-
-    def __init__(self, key="owners", manager_cls=None):
-        """Constructor."""
-        self._manager_cls = manager_cls or OwnerRelationManager
-        super().__init__(key=key)
-
-    @property
-    def mapping(self):
-        return {
-            self.attr_name: {
-                "type": "object",
-                "properties": {"user": {"type": "keyword", "ignore_above": 256}},
-            },
-        }
-
-    def pre_commit(self, record):
-        """Commit the communities field."""
-        manager = self.obj(record)
-        self.set_dictkey(record, manager.to_dict())
-
-    def pre_dump(self, record, data, dumper=None):
-        """Called before a record is dumped."""
-        # parent record commit op is not called during update, resulting in the parent not being converted correctly into 'dict', ie. the dict() function in invenio_records.dumpers.base #36 works incorrectly
-        manager = self.obj(record)
-        self.set_dictkey(record, manager.to_dict())
-
-    def obj(self, record):
-        """Get or crate the communities manager."""
-        # Check cache
-        obj = self._get_cache(record)
-        if obj is not None:
-            return obj
-
-        serialized_owners = self.get_dictkey(record)
-        # Create manager
-        obj = self._manager_cls(record.id, serialized_owners)
-        self._set_cache(record, obj)
-        return obj
-
-    def __get__(self, record, owner=None):
-        """Get the persistent identifier."""
-        if record is None:
-            return self
-        return self.obj(record)

oarepo_runtime/records/systemfields/record_status.py

@@ -1,35 +0,0 @@
-from invenio_records.systemfields import SystemField
-
-from .mapping import MappingSystemFieldMixin
-
-
-class RecordStatusResult:
-    def __init__(self, record, attr_name):
-        self.record = record
-        self.attr_name = attr_name
-
-
-class RecordStatusSystemField(MappingSystemFieldMixin, SystemField):
-    @property
-    def mapping(self):
-        return {
-            self.attr_name: {
-                "type": "keyword",
-            },
-        }
-
-    def search_load(self, data, record_cls):
-        data.pop(self.attr_name, None)
-
-    def search_dump(self, data, record):
-        if getattr(record, "is_draft"):
-            data[self.attr_name] = "draft"
-        else:
-            data[self.attr_name] = "published"
-
-    def __get__(self, record, owner=None):
-        """Accessing the attribute."""
-        # Class access
-        if record is None:
-            return self
-        return RecordStatusResult(record, self.attr_name)

oarepo_runtime/records/systemfields/selectors.py

@@ -1,98 +0,0 @@
-import dataclasses
-from typing import Any, Callable, List, Protocol, Tuple
-
-from oarepo_runtime.records.relations.lookup import lookup_key
-
-
-class Selector(Protocol):
-    def select(self, record) -> List[Any]:
-        return []
-
-
-class PathSelector(Selector):
-    def __init__(self, *paths):
-        self.paths = [x.split(".") for x in paths]
-
-    def select(self, record):
-        ret = []
-        for path in self.paths:
-            for rec in getter(record, path):
-                ret.append(rec)
-        return ret
-
-
-class FirstItemSelector(PathSelector):
-    def select(self, record):
-        for rec in super().select(record):
-            return [rec]
-        return []
-
-
-@dataclasses.dataclass
-class FilteredSelector(Selector):
-    """
-    Selector which filters output of another selector
-    Example:
-        FilteredSelector(PathSelector("metadata.creators", "metadata.contributors"),
-                                         filter=lambda x: x["nameType"] == "personal", projection="affiliations")
-
-    selects affiliations of creators with nameType personal from following data
-
-    data = {
-        "metadata": {
-            "creators": [
-                {"name": "hugo", "affiliations": ["uni1", "uni2"], "nameType": "personal"},
-                {"name": "uni3", "nameType": "organizational"},
-            ]
-        }
-    }
-    """
-    selector: Selector
-    filter: Callable[[Any], bool]
-    projection: Callable[[Any], Any] | str = None
-
-    def select(self, record):
-        selected = self.selector.select(record)
-        selected = filter(self.filter, selected)
-        if self.projection:
-            ret = []
-            for select_element in selected:
-                if isinstance(self.projection, str):
-                    result = [x.value for x in lookup_key(select_element, self.projection)]
-                else:
-                    result = self.projection(select_element)
-                if isinstance(result, list):
-                    ret += result
-                else:
-                    ret.append(result)
-        else:
-            ret = list(selected)
-        return ret
-
-
-@dataclasses.dataclass
-class MultiSelector(Selector):
-    """Selector concatenating outputs of multiple selectors"""
-
-    def __init__(self, *selectors: Selector):
-        self.selectors = selectors
-
-    def select(self, record):
-        ret = []
-        for selector in self.selectors:
-            ret += selector.select(record)
-        return ret
-
-
-def getter(data, path: List):
-    if len(path) == 0:
-        if isinstance(data, list):
-            yield from data
-        else:
-            yield data
-    elif isinstance(data, dict):
-        if path[0] in data:
-            yield from getter(data[path[0]], path[1:])
-    elif isinstance(data, list):
-        for item in data:
-            yield from getter(item, path)