invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/services/custom_fields/subject.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+
+"""Custom fields."""
+
+from invenio_i18n import lazy_gettext as _
+
+from ...contrib.subjects.api import Subject
+from ...contrib.subjects.schema import SubjectRelationSchema
+from .vocabulary import VocabularyCF
+
+
+class SubjectCF(VocabularyCF):
+    """Custom field for subjects."""
+
+    field_keys = ["id", "subject"]
+
+    def __init__(self, **kwargs):
+        """Constructor."""
+        super().__init__(
+            vocabulary_id="subjects",
+            schema=SubjectRelationSchema,
+            ui_schema=SubjectRelationSchema,
+            **kwargs,
+        )
+        self.pid_field = Subject.pid
+
+    @property
+    def mapping(self):
+        """Return the mapping."""
+        _mapping = {
+            "type": "object",
+            "properties": {
+                "@v": {"type": "keyword"},
+                "id": {"type": "keyword"},
+                "subject": {"type": "keyword"},
+            },
+        }
+
+        return _mapping
+
+
+SUBJECT_FIELDS_UI = [
+    {
+        "section": _("Subjects"),
+        "fields": [
+            dict(
+                field="subjects",
+                ui_widget="SubjectAutocompleteDropdown",
+                isGenericVocabulary=False,
+                props=dict(
+                    label=_("Keywords and subjects"),
+                    icon="tag",
+                    description=_("The subjects related to the community"),
+                    placeholder=_("Search for a subject by name e.g. Psychology ..."),
+                    autocompleteFrom="api/subjects",
+                    noQueryMessage=_("Search for subjects..."),
+                    autocompleteFromAcceptHeader="application/vnd.inveniordm.v1+json",
+                    required=False,
+                    multiple=True,
+                    clearable=True,
+                    allowAdditions=False,
+                ),
+                template="invenio_vocabularies/subjects.html",
+            )
+        ],
+    }
+]
+
+
+SUBJECT_FIELDS = {
+    SubjectCF(
+        name="subjects",
+        multiple=True,
+        dump_options=False,
+    )
+}

invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Custom Fields for InvenioRDM."""
+
+from invenio_records_resources.records.systemfields import PIDListRelation, PIDRelation
+from invenio_records_resources.services.custom_fields.base import BaseCF
+from marshmallow import fields
+
+from ...proxies import current_service
+from ...records.api import Vocabulary
+from ...resources.serializer import VocabularyL10NItemSchema
+from ...services.schema import VocabularyRelationSchema
+
+
+class VocabularyCF(BaseCF):
+    """Vocabulary custom field.
+
+    Supporting common vocabulary structure.
+    """
+
+    field_keys = ["id", "props", "title", "icon"]
+    """Return field's keys for querying.
+
+    These keys are used to select which information to return from the
+    vocabulary that is queried.
+    """
+
+    def __init__(
+        self,
+        name,
+        vocabulary_id,
+        multiple=False,
+        dump_options=True,
+        sort_by=None,
+        schema=VocabularyRelationSchema,
+        ui_schema=VocabularyL10NItemSchema,
+        **kwargs,
+    ):
+        """Constructor."""
+        super().__init__(name, **kwargs)
+        self.relation_cls = PIDRelation if not multiple else PIDListRelation
+        self.vocabulary_id = vocabulary_id
+        self.dump_options = dump_options
+        self.multiple = multiple
+        self.sort_by = sort_by
+        self.schema = schema
+        self.ui_schema = ui_schema
+        self.pid_field = Vocabulary.pid.with_type_ctx(self.vocabulary_id)
+
+    @property
+    def mapping(self):
+        """Return the mapping."""
+        _mapping = {
+            "type": "object",
+            "properties": {
+                "@v": {"type": "keyword"},
+                "id": {"type": "keyword"},
+                "title": {"type": "object", "dynamic": "true"},
+            },
+        }
+
+        return _mapping
+
+    @property
+    def field(self):
+        """Marshmallow schema for vocabulary custom fields."""
+        return fields.Nested(self.schema, many=self.multiple, **self._field_args)
+
+    @property
+    def ui_field(self):
+        """Marshmallow UI schema for vocabulary custom fields.
+
+        This schema is used in the UIJSONSerializer and controls how the field will be
+        dumped in the UI. It takes responsibility of the localization of strings.
+        """
+        return fields.Nested(self.ui_schema, many=self.multiple, **self._field_args)
+
+    def options(self, identity):
+        """Return UI serialized vocabulary items."""
+        if self.dump_options:
+            vocabs = current_service.read_all(
+                identity,
+                fields=self.field_keys,
+                type=self.vocabulary_id,
+                sort=self.sort_by,
+            )
+            options = []
+            for vocab in vocabs:
+                options.append(VocabularyL10NItemSchema().dump(vocab))
+
+            return options

invenio_vocabularies/services/facets.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2023 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary facets."""
+
+from flask_principal import AnonymousIdentity
+from invenio_cache.decorators import cached_with_expiration
+from invenio_i18n.ext import current_i18n
+from invenio_records_resources.proxies import current_service_registry
+from invenio_records_resources.services.errors import FacetNotFoundError
+from marshmallow_utils.fields.babel import gettext_from_dict
+from speaklater import make_lazy_string
+from sqlalchemy.exc import NoResultFound
+
+from ..proxies import current_service
+
+
+def get_service(service_id):
+    """Get the service object by name.
+
+    It is required to access the registry lazily, to avoid "out of
+    application context" errors.
+    """
+    return current_service_registry.get(service_id) if service_id else current_service
+
+
+def get_vocabs(service_id, type, fields, ids):
+    """Fetch vocabulary values by ids, using the service."""
+    service = get_service(service_id)
+    vocabs = service.read_many(
+        AnonymousIdentity(), type=type, ids=list(ids), fields=list(fields)
+    )
+    return list(vocabs.hits)  # the service returns a generator
+
+
+@cached_with_expiration
+def get_cached_vocab(service_id, type, fields, id_):
+    """Cache vocabulary values by type in-memory."""
+    vocabs = get_vocabs(service_id, type, fields, [id_])
+    return vocabs[0] if vocabs else None
+
+
+def lazy_get_label(vocab_item):
+    """Lazy evaluation of a localized vocabulary label."""
+    params = {"locale": current_i18n.locale, "default_locale": "en"}
+
+    return make_lazy_string(gettext_from_dict, vocab_item, **params)
+
+
+class VocabularyLabels:
+    """Fetching of vocabulary labels for facets."""
+
+    def __init__(
+        self, vocabulary, cache=True, cache_ttl=3600, service_id=None, id_field="id"
+    ):
+        """Initialize the labels.
+
+        :param vocabulary: the name of the vocabulary type.
+        :param cache: use simple process in-memory cache when True.
+        :param cache_ttl: cache expiration in seconds.
+        :param service_id: the id of the registered service to be used
+            when fetching values for the vocabulary.
+        :param id_field: the name of the `id` field.
+        """
+        self.vocabulary = vocabulary
+        self.cache = cache
+        self.cache_ttl = cache_ttl
+        self.fields = ("id", "title")  # not configurable
+        self.service_id = service_id
+        self.id_field = id_field
+
+    def _vocab_to_label(self, vocab):
+        """Returns the label string for a vocabulary entry."""
+        return lazy_get_label(vocab["title"])
+
+    def __call__(self, ids):
+        """Return the mapping when evaluated."""
+        if not ids:
+            return {}
+
+        labels = {}
+        try:
+            if self.cache:
+                for id_ in ids:
+                    vocab = get_cached_vocab(
+                        self.service_id,
+                        self.vocabulary,
+                        self.fields,
+                        id_,
+                        cache_ttl=self.cache_ttl,
+                    )
+                    if not vocab:
+                        continue
+                    labels[vocab[self.id_field]] = self._vocab_to_label(vocab)
+            else:
+                vocab_list = get_vocabs(
+                    self.service_id,
+                    self.vocabulary,
+                    self.fields,
+                    ids,
+                )
+                for vocab in vocab_list:
+                    id_ = vocab[self.id_field]
+                    if id_ in ids:
+                        labels[id_] = self._vocab_to_label(vocab)
+        except NoResultFound:
+            raise FacetNotFoundError(self.vocabulary)
+
+        return labels

invenio_vocabularies/services/generators.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+#
+
+"""Vocabulary generators."""
+
+from invenio_access import any_user, authenticated_user
+from invenio_records_permissions.generators import ConditionalGenerator
+from invenio_search.engine import dsl
+
+
+class IfTags(ConditionalGenerator):
+    """Generator to filter based on tags.
+
+    This generator will filter out records based on the tags field.
+    """
+
+    def __init__(self, tags, then_, else_):
+        """Constructor."""
+        self.tags = tags or []
+        super().__init__(then_, else_)
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record has the tags."""
+        return any(tag in record.get("tags", []) for tag in self.tags)
+
+    def query_filter(self, **kwargs):
+        """Search based on configured tags."""
+        must_not_clauses = [dsl.Q("terms", tags=self.tags)]
+        return dsl.Q(
+            "bool",
+            must_not=must_not_clauses,
+        )

invenio_vocabularies/services/permissions.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary permissions."""
+
+from invenio_records_permissions import RecordPermissionPolicy
+from invenio_records_permissions.generators import AnyUser, SystemProcess
+
+from invenio_vocabularies.services.generators import IfTags
+
+
+class PermissionPolicy(RecordPermissionPolicy):
+    """Permission policy."""
+
+    can_search = [SystemProcess(), AnyUser()]
+    can_read = [
+        SystemProcess(),
+        IfTags(["unlisted"], then_=[SystemProcess()], else_=[AnyUser()]),
+    ]
+    can_create = [SystemProcess()]
+    can_update = [SystemProcess()]
+    can_delete = [SystemProcess()]
+    can_manage = [SystemProcess()]
+    # this permission is needed for the /api/vocabularies/ endpoint
+    can_list_vocabularies = [SystemProcess(), AnyUser()]

invenio_vocabularies/services/querystr.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Querystring parsing."""
+
+import warnings
+from functools import partial
+
+from invenio_records_resources.services.records.params import SuggestQueryParser
+from invenio_search.engine import dsl
+
+
+class FilteredSuggestQueryParser(SuggestQueryParser):
+    """Query parser for filtered search-as-you-type/auto completion."""
+
+    @classmethod
+    def factory(cls, filter_field=None, **extra_params):
+        """Create a prepared instance of the query parser."""
+        warnings.warn(
+            "FilteredSuggestQueryParser is deprecated, use SuggestQueryParser or CompositeSuggestQueryParser instead",
+            DeprecationWarning,
+        )
+        return partial(cls, filter_field=filter_field, extra_params=extra_params)
+
+    def __init__(self, identity=None, filter_field=None, extra_params=None):
+        """Constructor."""
+        super().__init__(identity=identity, extra_params=extra_params)
+        self.filter_field = filter_field
+
+    def parse(self, query_str):
+        """Parse the query."""
+        subtype_s, query_str = self.extract_subtype_s(query_str)
+        query = super().parse(query_str)
+        if subtype_s:
+            query = query & dsl.Q("terms", **{self.filter_field: subtype_s})
+        return query
+
+    def extract_subtype_s(self, query_str):
+        """Extract the filtering subtype(s) from query_str.
+
+        Return (<subtypes>, <rest of original query string>).
+        """
+        parts = query_str.split(":", 1)
+        if len(parts) == 1:
+            subtypes = []
+            rest_query_str = query_str
+        else:
+            # Simplification: we can enforce no comma in subtype at
+            #                 subtype creation
+            subtypes = parts[0].split(",")
+            rest_query_str = parts[1]
+        return (subtypes, rest_query_str)

invenio_vocabularies/services/results.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 Uni Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary results."""
+
+from flask import current_app
+from invenio_records_resources.proxies import current_service_registry
+from invenio_records_resources.services import RecordServiceConfig
+from invenio_records_resources.services.records.results import RecordList
+from invenio_search import current_search_client
+
+from ..proxies import current_service
+
+
+class VocabularyTypeList(RecordList):
+    """Ensures that vocabulary type metadata is returned in the proper format."""
+
+    @property
+    def total(self):
+        """Get total number of hits."""
+        return self._results.total
+
+    @property
+    def custom_vocabulary_names(self):
+        """Checks whether vocabulary is a custom vocabulary."""
+        return current_app.config.get("VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [])
+
+    def to_dict(self):
+        """Formats result to a dict of hits."""
+        config_vocab_types = current_app.config.get(
+            "INVENIO_VOCABULARY_TYPE_METADATA", {}
+        )
+
+        count_terms_agg = {}
+        generic_stats = self._generic_vocabulary_statistics()
+        custom_stats = self._custom_vocabulary_statistics()
+
+        for k in generic_stats.keys() | custom_stats.keys():
+            count_terms_agg[k] = generic_stats.get(k, 0) + custom_stats.get(k, 0)
+
+        hits = self._results.items
+
+        # Extend database data with configuration & aggregation data.
+        results = []
+        for db_vocab_type in hits:
+            result = {
+                "id": db_vocab_type.id,
+                "pid_type": db_vocab_type.pid_type,
+                "count": count_terms_agg.get(db_vocab_type.id, 0),
+                "is_custom_vocabulary": db_vocab_type.id
+                in self.custom_vocabulary_names,
+            }
+
+            if db_vocab_type.id in config_vocab_types:
+                for k, v in config_vocab_types[db_vocab_type.id].items():
+                    result[k] = v
+
+            results.append(result)
+
+        for hit in results:
+            if self._links_item_tpl:
+                hit["links"] = self._links_item_tpl.expand(self._identity, hit)
+
+        res = {
+            "hits": {
+                "hits": results,
+                "total": self.total,
+            }
+        }
+
+        if self._params:
+            if self._links_tpl:
+                res["links"] = self._links_tpl.expand(self._identity, self.pagination)
+
+        return res
+
+    def _custom_vocabulary_statistics(self):
+        # query database for count of terms in custom vocabularies
+        returndict = {}
+        for vocab_type in self.custom_vocabulary_names:
+            custom_service = current_service_registry.get(vocab_type)
+            record_cls = custom_service.config.record_cls
+            returndict[vocab_type] = record_cls.model_cls.query.count()
+
+        return returndict
+
+    def _generic_vocabulary_statistics(self):
+        # Opensearch query for generic vocabularies
+        config: RecordServiceConfig = (
+            current_service.config
+        )  # TODO: Where to get the config from here? current_service is None
+        search_opts = config.search
+
+        search = search_opts.search_cls(
+            using=current_search_client,
+            index=config.record_cls.index.search_alias,
+        )
+
+        search.aggs.bucket("vocabularies", {"terms": {"field": "type.id", "size": 100}})
+
+        search_result = search.execute()
+        buckets = search_result.aggs.to_dict()["vocabularies"]["buckets"]
+
+        return {bucket["key"]: bucket["doc_count"] for bucket in buckets}

invenio_vocabularies/services/schema.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2025 Graz University of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary service schema."""
+
+from invenio_i18n import lazy_gettext as _
+from invenio_records_resources.services.records.schema import BaseRecordSchema
+from marshmallow import (
+    Schema,
+    ValidationError,
+    fields,
+    post_load,
+    pre_dump,
+    pre_load,
+    validate,
+    validates_schema,
+)
+from marshmallow_utils.fields import SanitizedUnicode
+
+i18n_strings = fields.Dict(
+    allow_none=False,
+    keys=fields.Str(validate=validate.Regexp("^[a-z]{2}$")),
+    values=fields.Str(),
+)
+"""Field definition for language aware strings."""
+
+
+class BaseVocabularyRelationSchema(Schema):
+    """Base Vocabulary relation schema."""
+
+    id = SanitizedUnicode(required=True)
+
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
+
+class VocabularyRelationSchema(BaseVocabularyRelationSchema):
+    """Vocabulary relation schema."""
+
+    title = fields.Dict(dump_only=True)
+
+    @pre_load
+    def clean(self, data, **kwargs):
+        """Removes dump_only fields.
+
+        Why: We want to allow the output of a Schema dump, to be a valid input
+             to a Schema load without causing strange issues.
+        """
+        value_is_dict = isinstance(data, dict)
+        if value_is_dict:
+            for name, field in self.fields.items():
+                if field.dump_only:
+                    data.pop(name, None)
+        return data
+
+
+class ContribVocabularyRelationSchema(Schema):
+    """Base Vocabulary relation schema."""
+
+    id = SanitizedUnicode()
+    ftf_name = None  # free text field name
+    parent_field_name = None
+
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
+    @validates_schema
+    def validate_relation_schema(self, data, **kwargs):
+        """Validates that either id either the free text field are present."""
+        id_ = data.get("id")
+        free_text = data.get(self.ftf_name)
+        if id_:
+            data = {"id": id_}
+        elif free_text:
+            data = {self.ftf_name: free_text}
+
+        if not id_ and not free_text:
+            raise ValidationError(
+                _(
+                    "An existing id or a free text {ftf_name} must be present.".format(
+                        ftf_name=self.ftf_name
+                    )
+                ),
+                self.parent_field_name,
+            )
+
+
+class BaseVocabularySchema(BaseRecordSchema):
+    """Base schema for vocabularies."""
+
+    title = i18n_strings
+    description = i18n_strings
+    icon = fields.Str(allow_none=False)
+    tags = fields.List(SanitizedUnicode())
+
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
+
+class VocabularySchema(BaseVocabularySchema):
+    """Service schema for vocabulary records."""
+
+    props = fields.Dict(allow_none=False, keys=fields.Str(), values=fields.Str())
+    type = fields.Str(attribute="type.id", required=True)
+
+
+class ModePIDFieldVocabularyMixin:
+    """Mixin for vocabularies using a model field for their PID."""
+
+    @validates_schema
+    def validate_id(self, data, **kwargs):
+        """Validates ID."""
+        is_create = "record" not in self.context
+        if is_create and "id" not in data:
+            raise ValidationError(_("Missing PID."), "id")
+        if not is_create:
+            data.pop("id", None)
+
+    @post_load(pass_many=False)
+    def move_id(self, data, **kwargs):
+        """Moves id to pid."""
+        if "id" in data:
+            data["pid"] = data.pop("id")
+        return data
+
+    @pre_dump(pass_many=False)
+    def extract_pid_value(self, data, **kwargs):
+        """Extracts the PID value."""
+        data["id"] = data.pid.pid_value
+        return data
+
+
+class DatastreamObject(Schema):
+    """Datastream object (reader, transformer, writer)."""
+
+    type = fields.Str(required=True)
+    args = fields.Dict(keys=fields.Str(), values=fields.Raw)
+
+
+class TaskSchema(Schema):
+    """Service schema for vocabulary tasks."""
+
+    readers = fields.List(
+        fields.Nested(DatastreamObject),
+        validate=validate.Length(min=1),
+        required=True,
+    )
+    transformers = fields.List(
+        fields.Nested(DatastreamObject),
+        validate=validate.Length(min=1),
+        required=False,
+    )
+    writers = fields.List(
+        fields.Nested(DatastreamObject),
+        validate=validate.Length(min=1),
+        required=True,
+    )