invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -12,6 +12,7 @@ from invenio_records_resources.services.custom_fields.base import BaseCF
 from marshmallow import fields
 
 from ...proxies import current_service
+from ...records.api import Vocabulary
 from ...resources.serializer import VocabularyL10NItemSchema
 from ...services.schema import VocabularyRelationSchema
 
@@ -30,7 +31,15 @@ class VocabularyCF(BaseCF):
     """
 
     def __init__(
-        self, name, vocabulary_id, multiple=False, dump_options=True, **kwargs
+        self,
+        name,
+        vocabulary_id,
+        multiple=False,
+        dump_options=True,
+        sort_by=None,
+        schema=VocabularyRelationSchema,
+        ui_schema=VocabularyL10NItemSchema,
+        **kwargs,
     ):
         """Constructor."""
         super().__init__(name, **kwargs)
@@ -38,6 +47,10 @@ class VocabularyCF(BaseCF):
         self.vocabulary_id = vocabulary_id
         self.dump_options = dump_options
         self.multiple = multiple
+        self.sort_by = sort_by
+        self.schema = schema
+        self.ui_schema = ui_schema
+        self.pid_field = Vocabulary.pid.with_type_ctx(self.vocabulary_id)
 
     @property
     def mapping(self):
@@ -47,7 +60,7 @@ class VocabularyCF(BaseCF):
             "properties": {
                 "@v": {"type": "keyword"},
                 "id": {"type": "keyword"},
-                "title": {"type": "object", "dynamic": True},
+                "title": {"type": "object", "dynamic": "true"},
             },
         }
 
@@ -56,9 +69,7 @@ class VocabularyCF(BaseCF):
     @property
     def field(self):
         """Marshmallow schema for vocabulary custom fields."""
-        return fields.Nested(
-            VocabularyRelationSchema, many=self.multiple, **self._field_args
-        )
+        return fields.Nested(self.schema, many=self.multiple, **self._field_args)
 
     @property
     def ui_field(self):
@@ -67,9 +78,7 @@ class VocabularyCF(BaseCF):
         This schema is used in the UIJSONSerializer and controls how the field will be
         dumped in the UI. It takes responsibility of the localization of strings.
         """
-        return fields.Nested(
-            VocabularyL10NItemSchema, many=self.multiple, **self._field_args
-        )
+        return fields.Nested(self.ui_schema, many=self.multiple, **self._field_args)
 
     def options(self, identity):
         """Return UI serialized vocabulary items."""
@@ -78,6 +87,7 @@ class VocabularyCF(BaseCF):
                 identity,
                 fields=self.field_keys,
                 type=self.vocabulary_id,
+                sort=self.sort_by,
             )
             options = []
             for vocab in vocabs:

invenio_vocabularies/services/facets.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2023 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,6 +9,7 @@
 """Vocabulary facets."""
 
 from flask_principal import AnonymousIdentity
+from invenio_cache.decorators import cached_with_expiration
 from invenio_i18n.ext import current_i18n
 from invenio_records_resources.proxies import current_service_registry
 from invenio_records_resources.services.errors import FacetNotFoundError
@@ -19,6 +20,31 @@ from sqlalchemy.exc import NoResultFound
 from ..proxies import current_service
 
 
+def get_service(service_id):
+    """Get the service object by name.
+
+    It is required to access the registry lazily, to avoid "out of
+    application context" errors.
+    """
+    return current_service_registry.get(service_id) if service_id else current_service
+
+
+def get_vocabs(service_id, type, fields, ids):
+    """Fetch vocabulary values by ids, using the service."""
+    service = get_service(service_id)
+    vocabs = service.read_many(
+        AnonymousIdentity(), type=type, ids=list(ids), fields=list(fields)
+    )
+    return list(vocabs.hits)  # the service returns a generator
+
+
+@cached_with_expiration
+def get_cached_vocab(service_id, type, fields, id_):
+    """Cache vocabulary values by type in-memory."""
+    vocabs = get_vocabs(service_id, type, fields, [id_])
+    return vocabs[0] if vocabs else None
+
+
 def lazy_get_label(vocab_item):
     """Lazy evaluation of a localized vocabulary label."""
     params = {"locale": current_i18n.locale, "default_locale": "en"}
@@ -29,56 +55,60 @@ def lazy_get_label(vocab_item):
 class VocabularyLabels:
     """Fetching of vocabulary labels for facets."""
 
-    def __init__(self, vocabulary, cache=False, service_name=None, id_field="id"):
-        """Initialize the labels."""
+    def __init__(
+        self, vocabulary, cache=True, cache_ttl=3600, service_id=None, id_field="id"
+    ):
+        """Initialize the labels.
+
+        :param vocabulary: the name of the vocabulary type.
+        :param cache: use simple process in-memory cache when True.
+        :param cache_ttl: cache expiration in seconds.
+        :param service_id: the id of the registered service to be used
+            when fetching values for the vocabulary.
+        :param id_field: the name of the `id` field.
+        """
         self.vocabulary = vocabulary
         self.cache = cache
-        self.fields = ["id", "title"]  # not configurable
-        self.service_name = service_name
+        self.cache_ttl = cache_ttl
+        self.fields = ("id", "title")  # not configurable
+        self.service_id = service_id
         self.id_field = id_field
 
-    @property
-    def service(self):
-        """Service property.
-
-        It is required to access the regitry lazily to avoid out of
-        application context errors.
-        """
-        if not self.service_name:
-            return current_service
-        return current_service_registry.get(self.service_name)
-
     def _vocab_to_label(self, vocab):
         """Returns the label string for a vocabulary entry."""
         return lazy_get_label(vocab["title"])
 
     def __call__(self, ids):
         """Return the mapping when evaluated."""
-        identity = AnonymousIdentity()
+        if not ids:
+            return {}
+
+        labels = {}
         try:
-            if not self.cache:
-                vocabs = self.service.read_many(
-                    identity, type=self.vocabulary, ids=ids, fields=self.fields
-                )
+            if self.cache:
+                for id_ in ids:
+                    vocab = get_cached_vocab(
+                        self.service_id,
+                        self.vocabulary,
+                        self.fields,
+                        id_,
+                        cache_ttl=self.cache_ttl,
+                    )
+                    if not vocab:
+                        continue
+                    labels[vocab[self.id_field]] = self._vocab_to_label(vocab)
             else:
-                vocabs = self.service.read_all(
-                    identity, type=self.vocabulary, fields=self.fields
+                vocab_list = get_vocabs(
+                    self.service_id,
+                    self.vocabulary,
+                    self.fields,
+                    ids,
                 )
+                for vocab in vocab_list:
+                    id_ = vocab[self.id_field]
+                    if id_ in ids:
+                        labels[id_] = self._vocab_to_label(vocab)
         except NoResultFound:
             raise FacetNotFoundError(self.vocabulary)
 
-        labels = {}
-        vocab_list = list(vocabs.hits)  # the service returns a generator
-        ids = set(ids)
-        seen = set()
-        for vocab in vocab_list:
-            # cannot loop over ids because vocab id is inside each item
-            if len(ids) == len(seen):
-                break
-
-            id_ = vocab[self.id_field]
-            if id_ in ids:
-                labels[id_] = self._vocab_to_label(vocab)
-                seen.add(id_)
-
         return labels

invenio_vocabularies/services/permissions.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -21,3 +21,5 @@ class PermissionPolicy(RecordPermissionPolicy):
     can_update = [SystemProcess()]
     can_delete = [SystemProcess()]
     can_manage = [SystemProcess()]
+    # this permission is needed for the /api/vocabularies/ endpoint
+    can_list_vocabularies = [SystemProcess(), AnyUser()]

invenio_vocabularies/services/results.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 Uni Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary results."""
+
+from flask import current_app
+from invenio_records_resources.proxies import current_service_registry
+from invenio_records_resources.services import RecordServiceConfig
+from invenio_records_resources.services.records.results import RecordList
+from invenio_search import current_search_client
+
+from ..proxies import current_service
+
+
+class VocabularyTypeList(RecordList):
+    """Ensures that vocabulary type metadata is returned in the proper format."""
+
+    @property
+    def total(self):
+        """Get total number of hits."""
+        return self._results.total
+
+    @property
+    def custom_vocabulary_names(self):
+        """Checks whether vocabulary is a custom vocabulary."""
+        return current_app.config.get("VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [])
+
+    def to_dict(self):
+        """Formats result to a dict of hits."""
+        config_vocab_types = current_app.config.get(
+            "INVENIO_VOCABULARY_TYPE_METADATA", {}
+        )
+
+        count_terms_agg = {}
+        generic_stats = self._generic_vocabulary_statistics()
+        custom_stats = self._custom_vocabulary_statistics()
+
+        for k in generic_stats.keys() | custom_stats.keys():
+            count_terms_agg[k] = generic_stats.get(k, 0) + custom_stats.get(k, 0)
+
+        hits = self._results.items
+
+        # Extend database data with configuration & aggregation data.
+        results = []
+        for db_vocab_type in hits:
+            result = {
+                "id": db_vocab_type.id,
+                "pid_type": db_vocab_type.pid_type,
+                "count": count_terms_agg.get(db_vocab_type.id, 0),
+                "is_custom_vocabulary": db_vocab_type.id
+                in self.custom_vocabulary_names,
+            }
+
+            if db_vocab_type.id in config_vocab_types:
+                for k, v in config_vocab_types[db_vocab_type.id].items():
+                    result[k] = v
+
+            results.append(result)
+
+        for hit in results:
+            if self._links_item_tpl:
+                hit["links"] = self._links_item_tpl.expand(self._identity, hit)
+
+        res = {
+            "hits": {
+                "hits": results,
+                "total": self.total,
+            }
+        }
+
+        if self._params:
+            if self._links_tpl:
+                res["links"] = self._links_tpl.expand(self._identity, self.pagination)
+
+        return res
+
+    def _custom_vocabulary_statistics(self):
+        # query database for count of terms in custom vocabularies
+        returndict = {}
+        for vocab_type in self.custom_vocabulary_names:
+            custom_service = current_service_registry.get(vocab_type)
+            record_cls = custom_service.config.record_cls
+            returndict[vocab_type] = record_cls.model_cls.query.count()
+
+        return returndict
+
+    def _generic_vocabulary_statistics(self):
+        # Opensearch query for generic vocabularies
+        config: RecordServiceConfig = (
+            current_service.config
+        )  # TODO: Where to get the config from here? current_service is None
+        search_opts = config.search
+
+        search = search_opts.search_cls(
+            using=current_search_client,
+            index=config.record_cls.index.search_alias,
+        )
+
+        search.aggs.bucket("vocabularies", {"terms": {"field": "type.id", "size": 100}})
+
+        search_result = search.execute()
+        buckets = search_result.aggs.to_dict()["vocabularies"]["buckets"]
+
+        return {bucket["key"]: bucket["doc_count"] for bucket in buckets}

invenio_vocabularies/services/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -14,6 +14,8 @@ from marshmallow import (
     Schema,
     ValidationError,
     fields,
+    post_load,
+    pre_dump,
     pre_load,
     validate,
     validates_schema,
@@ -33,6 +35,9 @@ class BaseVocabularyRelationSchema(Schema):
 
     id = SanitizedUnicode(required=True)
 
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
 
 class VocabularyRelationSchema(BaseVocabularyRelationSchema):
     """Vocabulary relation schema."""
@@ -61,6 +66,9 @@ class ContribVocabularyRelationSchema(Schema):
     ftf_name = None  # free text field name
     parent_field_name = None
 
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
     @validates_schema
     def validate_relation_schema(self, data, **kwargs):
         """Validates that either id either the free text field are present."""
@@ -88,16 +96,45 @@ class BaseVocabularySchema(BaseRecordSchema):
     title = i18n_strings
     description = i18n_strings
     icon = fields.Str(allow_none=False)
+    tags = fields.List(SanitizedUnicode())
+
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
 
 
 class VocabularySchema(BaseVocabularySchema):
     """Service schema for vocabulary records."""
 
     props = fields.Dict(allow_none=False, keys=fields.Str(), values=fields.Str())
-    tags = fields.List(SanitizedUnicode())
     type = fields.Str(attribute="type.id", required=True)
 
 
+class ModePIDFieldVocabularyMixin:
+    """Mixin for vocabularies using a model field for their PID."""
+
+    @validates_schema
+    def validate_id(self, data, **kwargs):
+        """Validates ID."""
+        is_create = "record" not in self.context
+        if is_create and "id" not in data:
+            raise ValidationError(_("Missing PID."), "id")
+        if not is_create:
+            data.pop("id", None)
+
+    @post_load(pass_many=False)
+    def move_id(self, data, **kwargs):
+        """Moves id to pid."""
+        if "id" in data:
+            data["pid"] = data.pop("id")
+        return data
+
+    @pre_dump(pass_many=False)
+    def extract_pid_value(self, data, **kwargs):
+        """Extracts the PID value."""
+        data["id"] = data.pid.pid_value
+        return data
+
+
 class DatastreamObject(Schema):
     """Datastream object (reader, transformer, writer)."""
 

invenio_vocabularies/services/service.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,108 +10,59 @@
 
 """Vocabulary service."""
 
+import sqlalchemy as sa
 from invenio_cache import current_cache
-from invenio_db import db
-from invenio_i18n import lazy_gettext as _
-from invenio_records_resources.services import (
-    Link,
-    LinksTemplate,
-    RecordService,
-    RecordServiceConfig,
-    SearchOptions,
-    pagination_links,
-)
-from invenio_records_resources.services.records.components import DataComponent
-from invenio_records_resources.services.records.params import (
-    FilterParam,
-    SuggestQueryParser,
-)
+from invenio_records_resources.services import LinksTemplate, RecordService
+from invenio_records_resources.services.base.utils import map_search_params
 from invenio_records_resources.services.records.schema import ServiceSchemaWrapper
 from invenio_records_resources.services.uow import unit_of_work
 from invenio_search.engine import dsl
 
-from ..records.api import Vocabulary
 from ..records.models import VocabularyType
-from .components import PIDComponent, VocabularyTypeComponent
-from .permissions import PermissionPolicy
-from .schema import TaskSchema, VocabularySchema
 from .tasks import process_datastream
 
 
-class VocabularySearchOptions(SearchOptions):
-    """Search options."""
-
-    params_interpreters_cls = [
-        FilterParam.factory(param="tags", field="tags"),
-    ] + SearchOptions.params_interpreters_cls
-
-    suggest_parser_cls = SuggestQueryParser.factory(
-        fields=[
-            "id.text^100",
-            "id.text._2gram",
-            "id.text._3gram",
-            "title.en^5",
-            "title.en._2gram",
-            "title.en._3gram",
-        ],
-    )
-
-    sort_default = "bestmatch"
-
-    sort_default_no_query = "title"
-
-    sort_options = {
-        "bestmatch": dict(
-            title=_("Best match"),
-            fields=["_score"],  # ES defaults to desc on `_score` field
-        ),
-        "title": dict(
-            title=_("Title"),
-            fields=["title_sort"],
-        ),
-        "newest": dict(
-            title=_("Newest"),
-            fields=["-created"],
-        ),
-        "oldest": dict(
-            title=_("Oldest"),
-            fields=["created"],
-        ),
-    }
-
-
-class VocabulariesServiceConfig(RecordServiceConfig):
-    """Vocabulary service configuration."""
-
-    service_id = "vocabularies"
-    indexer_queue_name = "vocabularies"
-    permission_policy_cls = PermissionPolicy
-    record_cls = Vocabulary
-    schema = VocabularySchema
-    task_schema = TaskSchema
-
-    search = VocabularySearchOptions
-
-    components = [
-        # Order of components are important!
-        VocabularyTypeComponent,
-        DataComponent,
-        PIDComponent,
-    ]
-
-    links_item = {
-        "self": Link(
-            "{+api}/vocabularies/{type}/{id}",
-            vars=lambda record, vars: vars.update(
-                {
-                    "id": record.pid.pid_value,
-                    "type": record.type.id,
-                }
-            ),
-        ),
-    }
+class VocabularyTypeService(RecordService):
+    """Vocabulary type service."""
+
+    def rebuild_index(self, identity, uow=None):
+        """Raise error since services are not backed by search indices."""
+        raise NotImplementedError()
+
+    def search(self, identity, params=None):
+        """Search for vocabulary types entries."""
+        self.require_permission(identity, "list_vocabularies")
+
+        search_params = map_search_params(self.config.search, params)
+
+        query_param = search_params["q"]
+
+        filters = []
+        if query_param:
+            filters.extend([VocabularyType.id.ilike(f"%{query_param}%")])
+
+        vocabulary_types = (
+            VocabularyType.query.filter(sa.or_(*filters))
+            .order_by(
+                search_params["sort_direction"](
+                    sa.text(",".join(search_params["sort"]))
+                )
+            )
+            .paginate(
+                page=search_params["page"],
+                per_page=search_params["size"],
+                error_out=False,
+            )
+        )
 
-    links_search = pagination_links("{+api}/vocabularies/{type}{?args*}")
+        return self.config.result_list_cls(
+            self,
+            identity,
+            vocabulary_types,
+            search_params,
+            links_tpl=LinksTemplate(self.config.links_search, context={"args": params}),
+            links_item_tpl=self.links_item_tpl,
+        )
 
 
 class VocabulariesService(RecordService):
@@ -145,7 +97,7 @@ class VocabulariesService(RecordService):
             params,
             search_preference,
             extra_filter=dsl.Q("term", type__id=vocabulary_type.id),
-            **kwargs
+            **kwargs,
         ).execute()
 
         return self.result_list(

invenio_vocabularies/services/tasks.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.

invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/subjects.html

@@ -0,0 +1,23 @@
+{#
+  Copyright (C) 2024 CERN.
+
+  Invenio RDM Records is free software; you can redistribute it and/or modify
+  it under the terms of the MIT License; see LICENSE file for more details.
+#}
+
+{%- set subjects = field_value -%}
+{% if subjects %}
+  <ul class="ui horizontal list no-bullets subjects">
+    {%- for subject in subjects %}
+    <li class="item">
+      {%- set q = 'metadata.subjects.subject:"' + subject.subject + '"' %}
+      <a href="{{ url_for('invenio_search_ui.search', q=q) }}"
+          class="subject"
+          title="{{ _('Search results for ') + subject.subject }}"
+      >
+        {{ subject.subject }}
+      </a>
+    </li>
+    {%- endfor %}
+  </ul>
+{% endif %}

invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabularies-list.html

@@ -0,0 +1,12 @@
+{#
+    Copyright (C) 2024 Uni Münster.
+
+    Invenio App RDM is free software; you can redistribute it and/or modify it
+    under the terms of the MIT License; see LICENSE file for more details.
+  #}
+  {% extends "invenio_administration/search.html" %}
+
+  {% block javascript %}
+  {{ super() }}
+  {{ webpack['invenio-vocabularies-search.js'] }}
+{% endblock %}