invenio-vocabularies 4.0.0__py2.py3-none-any.whl → 4.2.0__py2.py3-none-any.whl

invenio_vocabularies/services/config.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 Uni Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary services configs."""
+
+import sqlalchemy as sa
+from flask import current_app
+from invenio_i18n import lazy_gettext as _
+from invenio_records_resources.services import (
+    Link,
+    LinksTemplate,
+    RecordService,
+    RecordServiceConfig,
+    SearchOptions,
+    pagination_links,
+)
+from invenio_records_resources.services.base import (
+    ConditionalLink,
+    Service,
+    ServiceListResult,
+)
+from invenio_records_resources.services.records.components import DataComponent
+from invenio_records_resources.services.records.params import (
+    FilterParam,
+    SuggestQueryParser,
+)
+
+from ..records.api import Vocabulary
+from ..records.models import VocabularyType
+from . import results
+from .components import PIDComponent, VocabularyTypeComponent
+from .permissions import PermissionPolicy
+from .schema import TaskSchema, VocabularySchema
+
+
+def is_custom_vocabulary_type(vocabulary_type, context):
+    """Check if the vocabulary type is a custom vocabulary type."""
+    return vocabulary_type["id"] in current_app.config.get(
+        "VOCABULARIES_CUSTOM_VOCABULARY_TYPES", []
+    )
+
+
+class VocabularySearchOptions(SearchOptions):
+    """Search options for vocabularies."""
+
+    params_interpreters_cls = [
+        FilterParam.factory(param="tags", field="tags"),
+    ] + SearchOptions.params_interpreters_cls
+
+    suggest_parser_cls = SuggestQueryParser.factory(
+        fields=[
+            "id.text^100",
+            "id.text._2gram",
+            "id.text._3gram",
+            "title.en^5",
+            "title.en._2gram",
+            "title.en._3gram",
+        ],
+    )
+
+    sort_default = "bestmatch"
+
+    sort_default_no_query = "title"
+
+    sort_options = {
+        "bestmatch": dict(
+            title=_("Best match"),
+            fields=["_score"],  # ES defaults to desc on `_score` field
+        ),
+        "title": dict(
+            title=_("Title"),
+            fields=["title_sort"],
+        ),
+        "newest": dict(
+            title=_("Newest"),
+            fields=["-created"],
+        ),
+        "oldest": dict(
+            title=_("Oldest"),
+            fields=["created"],
+        ),
+    }
+
+
+class VocabularyTypeSearchOptions(SearchOptions):
+    """Search options for vocabulary types."""
+
+    sort_options = {
+        "id": dict(
+            title=_("ID"),
+            fields=["id"],
+        ),
+    }
+
+    sort_default = "id"
+
+    sort_default_no_query = "id"
+
+    sort_direction_options = {
+        "asc": dict(title=_("Ascending"), fn=sa.asc),
+        "desc": dict(title=_("Descending"), fn=sa.desc),
+    }
+
+    sort_direction_default = "asc"
+
+
+class VocabulariesServiceConfig(RecordServiceConfig):
+    """Vocabulary service configuration."""
+
+    service_id = "vocabularies"
+    indexer_queue_name = "vocabularies"
+    permission_policy_cls = PermissionPolicy
+    record_cls = Vocabulary
+    schema = VocabularySchema
+    task_schema = TaskSchema
+
+    search = VocabularySearchOptions
+
+    components = [
+        # Order of components are important!
+        VocabularyTypeComponent,
+        DataComponent,
+        PIDComponent,
+    ]
+
+    links_item = {
+        "self": Link(
+            "{+api}/vocabularies/{type}/{id}",
+            vars=lambda record, vars: vars.update(
+                {
+                    "id": record.pid.pid_value,
+                    "type": record.type.id,
+                }
+            ),
+        ),
+    }
+
+    links_search = pagination_links("{+api}/vocabularies/{type}{?args*}")
+
+
+class VocabularyTypesServiceConfig(RecordServiceConfig):
+    """Vocabulary types service configuration."""
+
+    service_id = "vocabulary_types"
+    permission_policy_cls = PermissionPolicy
+    record_cls = VocabularyType
+    schema = VocabularySchema  # Works but should be VocabularyTypeSchema if this is defined at some point
+    result_list_cls = results.VocabularyTypeList
+
+    links_item = {
+        "self": ConditionalLink(
+            cond=is_custom_vocabulary_type,
+            if_=Link(
+                "{+api}/{id}",
+                vars=lambda vocab_type, vars: vars.update({"id": vocab_type["id"]}),
+            ),
+            else_=Link(
+                "{+api}/vocabularies/{id}",
+                vars=lambda vocab_type, vars: vars.update({"id": vocab_type["id"]}),
+            ),
+        )
+    }
+
+    search = VocabularyTypeSearchOptions
+
+    components = [
+        # Order of components are important!
+        VocabularyTypeComponent,
+        DataComponent,
+        PIDComponent,
+    ]
+
+    links_search = pagination_links("{+api}/vocabularies/{type}{?args*}")

invenio_vocabularies/services/custom_fields/subject.py

@@ -7,6 +7,7 @@
 
 
 """Custom fields."""
+
 from invenio_i18n import lazy_gettext as _
 
 from ...contrib.subjects.api import Subject
@@ -25,7 +26,7 @@ class SubjectCF(VocabularyCF):
             vocabulary_id="subjects",
             schema=SubjectRelationSchema,
             ui_schema=SubjectRelationSchema,
-            **kwargs
+            **kwargs,
         )
         self.pid_field = Subject.pid
 

invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -39,7 +39,7 @@ class VocabularyCF(BaseCF):
         sort_by=None,
         schema=VocabularyRelationSchema,
         ui_schema=VocabularyL10NItemSchema,
-        **kwargs
+        **kwargs,
     ):
         """Constructor."""
         super().__init__(name, **kwargs)

invenio_vocabularies/services/permissions.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -21,3 +21,5 @@ class PermissionPolicy(RecordPermissionPolicy):
     can_update = [SystemProcess()]
     can_delete = [SystemProcess()]
     can_manage = [SystemProcess()]
+    # this permission is needed for the /api/vocabularies/ endpoint
+    can_list_vocabularies = [SystemProcess(), AnyUser()]

invenio_vocabularies/services/results.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 Uni Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary results."""
+
+from flask import current_app
+from invenio_records_resources.proxies import current_service_registry
+from invenio_records_resources.services import RecordServiceConfig
+from invenio_records_resources.services.records.results import RecordList
+from invenio_search import current_search_client
+
+from ..proxies import current_service
+
+
+class VocabularyTypeList(RecordList):
+    """Ensures that vocabulary type metadata is returned in the proper format."""
+
+    @property
+    def total(self):
+        """Get total number of hits."""
+        return self._results.total
+
+    @property
+    def custom_vocabulary_names(self):
+        """Checks whether vocabulary is a custom vocabulary."""
+        return current_app.config.get("VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [])
+
+    def to_dict(self):
+        """Formats result to a dict of hits."""
+        config_vocab_types = current_app.config.get(
+            "INVENIO_VOCABULARY_TYPE_METADATA", {}
+        )
+
+        count_terms_agg = {}
+        generic_stats = self._generic_vocabulary_statistics()
+        custom_stats = self._custom_vocabulary_statistics()
+
+        for k in generic_stats.keys() | custom_stats.keys():
+            count_terms_agg[k] = generic_stats.get(k, 0) + custom_stats.get(k, 0)
+
+        hits = self._results.items
+
+        # Extend database data with configuration & aggregation data.
+        results = []
+        for db_vocab_type in hits:
+            result = {
+                "id": db_vocab_type.id,
+                "pid_type": db_vocab_type.pid_type,
+                "count": count_terms_agg.get(db_vocab_type.id, 0),
+                "is_custom_vocabulary": db_vocab_type.id
+                in self.custom_vocabulary_names,
+            }
+
+            if db_vocab_type.id in config_vocab_types:
+                for k, v in config_vocab_types[db_vocab_type.id].items():
+                    result[k] = v
+
+            results.append(result)
+
+        for hit in results:
+            if self._links_item_tpl:
+                hit["links"] = self._links_item_tpl.expand(self._identity, hit)
+
+        res = {
+            "hits": {
+                "hits": results,
+                "total": self.total,
+            }
+        }
+
+        if self._params:
+            if self._links_tpl:
+                res["links"] = self._links_tpl.expand(self._identity, self.pagination)
+
+        return res
+
+    def _custom_vocabulary_statistics(self):
+        # query database for count of terms in custom vocabularies
+        returndict = {}
+        for vocab_type in self.custom_vocabulary_names:
+            custom_service = current_service_registry.get(vocab_type)
+            record_cls = custom_service.config.record_cls
+            returndict[vocab_type] = record_cls.model_cls.query.count()
+
+        return returndict
+
+    def _generic_vocabulary_statistics(self):
+        # Opensearch query for generic vocabularies
+        config: RecordServiceConfig = (
+            current_service.config
+        )  # TODO: Where to get the config from here? current_service is None
+        search_opts = config.search
+
+        search = search_opts.search_cls(
+            using=current_search_client,
+            index=config.record_cls.index.search_alias,
+        )
+
+        search.aggs.bucket("vocabularies", {"terms": {"field": "type.id", "size": 100}})
+
+        search_result = search.execute()
+        buckets = search_result.aggs.to_dict()["vocabularies"]["buckets"]
+
+        return {bucket["key"]: bucket["doc_count"] for bucket in buckets}

invenio_vocabularies/services/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -35,6 +35,9 @@ class BaseVocabularyRelationSchema(Schema):
 
     id = SanitizedUnicode(required=True)
 
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
 
 class VocabularyRelationSchema(BaseVocabularyRelationSchema):
     """Vocabulary relation schema."""
@@ -63,6 +66,9 @@ class ContribVocabularyRelationSchema(Schema):
     ftf_name = None  # free text field name
     parent_field_name = None
 
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
+
     @validates_schema
     def validate_relation_schema(self, data, **kwargs):
         """Validates that either id either the free text field are present."""
@@ -90,13 +96,16 @@ class BaseVocabularySchema(BaseRecordSchema):
     title = i18n_strings
     description = i18n_strings
     icon = fields.Str(allow_none=False)
+    tags = fields.List(SanitizedUnicode())
+
+    # Nested field type for administration UI form generation
+    administration_schema_type = "vocabulary"
 
 
 class VocabularySchema(BaseVocabularySchema):
     """Service schema for vocabulary records."""
 
     props = fields.Dict(allow_none=False, keys=fields.Str(), values=fields.Str())
-    tags = fields.List(SanitizedUnicode())
     type = fields.Str(attribute="type.id", required=True)
 
 

invenio_vocabularies/services/service.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 Uni Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,9 +10,8 @@
 
 """Vocabulary service."""
 
+import sqlalchemy as sa
 from invenio_cache import current_cache
-from invenio_db import db
-from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import (
     Link,
     LinksTemplate,
@@ -20,97 +20,52 @@ from invenio_records_resources.services import (
     SearchOptions,
     pagination_links,
 )
-from invenio_records_resources.services.records.components import DataComponent
-from invenio_records_resources.services.records.params import (
-    FilterParam,
-    SuggestQueryParser,
-)
+from invenio_records_resources.services.base.utils import map_search_params
 from invenio_records_resources.services.records.schema import ServiceSchemaWrapper
 from invenio_records_resources.services.uow import unit_of_work
 from invenio_search.engine import dsl
 
-from ..records.api import Vocabulary
 from ..records.models import VocabularyType
-from .components import PIDComponent, VocabularyTypeComponent
-from .permissions import PermissionPolicy
-from .schema import TaskSchema, VocabularySchema
 from .tasks import process_datastream
 
 
-class VocabularySearchOptions(SearchOptions):
-    """Search options."""
-
-    params_interpreters_cls = [
-        FilterParam.factory(param="tags", field="tags"),
-    ] + SearchOptions.params_interpreters_cls
-
-    suggest_parser_cls = SuggestQueryParser.factory(
-        fields=[
-            "id.text^100",
-            "id.text._2gram",
-            "id.text._3gram",
-            "title.en^5",
-            "title.en._2gram",
-            "title.en._3gram",
-        ],
-    )
-
-    sort_default = "bestmatch"
-
-    sort_default_no_query = "title"
-
-    sort_options = {
-        "bestmatch": dict(
-            title=_("Best match"),
-            fields=["_score"],  # ES defaults to desc on `_score` field
-        ),
-        "title": dict(
-            title=_("Title"),
-            fields=["title_sort"],
-        ),
-        "newest": dict(
-            title=_("Newest"),
-            fields=["-created"],
-        ),
-        "oldest": dict(
-            title=_("Oldest"),
-            fields=["created"],
-        ),
-    }
-
-
-class VocabulariesServiceConfig(RecordServiceConfig):
-    """Vocabulary service configuration."""
-
-    service_id = "vocabularies"
-    indexer_queue_name = "vocabularies"
-    permission_policy_cls = PermissionPolicy
-    record_cls = Vocabulary
-    schema = VocabularySchema
-    task_schema = TaskSchema
-
-    search = VocabularySearchOptions
-
-    components = [
-        # Order of components are important!
-        VocabularyTypeComponent,
-        DataComponent,
-        PIDComponent,
-    ]
-
-    links_item = {
-        "self": Link(
-            "{+api}/vocabularies/{type}/{id}",
-            vars=lambda record, vars: vars.update(
-                {
-                    "id": record.pid.pid_value,
-                    "type": record.type.id,
-                }
-            ),
-        ),
-    }
+class VocabularyTypeService(RecordService):
+    """Vocabulary type service."""
+
+    def search(self, identity, params=None):
+        """Search for vocabulary types entries."""
+        self.require_permission(identity, "list_vocabularies")
+
+        search_params = map_search_params(self.config.search, params)
+
+        query_param = search_params["q"]
+
+        filters = []
+        if query_param:
+            filters.extend([VocabularyType.id.ilike(f"%{query_param}%")])
+
+        vocabulary_types = (
+            VocabularyType.query.filter(sa.or_(*filters))
+            .order_by(
+                search_params["sort_direction"](
+                    sa.text(",".join(search_params["sort"]))
+                )
+            )
+            .paginate(
+                page=search_params["page"],
+                per_page=search_params["size"],
+                error_out=False,
+            )
+        )
 
-    links_search = pagination_links("{+api}/vocabularies/{type}{?args*}")
+        return self.config.result_list_cls(
+            self,
+            identity,
+            vocabulary_types,
+            search_params,
+            links_tpl=LinksTemplate(self.config.links_search, context={"args": params}),
+            links_item_tpl=self.links_item_tpl,
+        )
 
 
 class VocabulariesService(RecordService):
@@ -145,7 +100,7 @@ class VocabulariesService(RecordService):
             params,
             search_preference,
             extra_filter=dsl.Q("term", type__id=vocabulary_type.id),
-            **kwargs
+            **kwargs,
         ).execute()
 
         return self.result_list(

invenio_vocabularies/services/tasks.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -11,7 +11,6 @@ from celery import shared_task
 from flask import current_app
 
 from ..datastreams.factories import DataStreamFactory
-from ..factories import get_vocabulary_config
 
 
 @shared_task(ignore_result=True)
@@ -27,32 +26,3 @@ def process_datastream(config):
         if result.errors:
             for err in result.errors:
                 current_app.logger.error(err)
-
-
-@shared_task()
-def import_funders():
-    """Import the funders vocabulary.
-
-    Only new records are imported.
-    Existing records are not updated.
-    """
-    vc = get_vocabulary_config("funders")
-    config = vc.get_config()
-
-    # When importing funders via a Celery task, make sure that we are automatically downloading the ROR file,
-    # instead of relying on a local file on the file system.
-    if config["readers"][0]["type"] == "ror-http":
-        readers_config_with_ror_http = config["readers"]
-    else:
-        readers_config_with_ror_http = [{"type": "ror-http"}] + config["readers"]
-
-    ds = DataStreamFactory.create(
-        readers_config=readers_config_with_ror_http,
-        transformers_config=config.get("transformers"),
-        writers_config=config["writers"],
-    )
-
-    for result in ds.process():
-        if result.errors:
-            for err in result.errors:
-                current_app.logger.error(err)

invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabularies-list.html

@@ -0,0 +1,12 @@
+{#
+    Copyright (C) 2024 Uni Münster.
+
+    Invenio App RDM is free software; you can redistribute it and/or modify it
+    under the terms of the MIT License; see LICENSE file for more details.
+  #}
+  {% extends "invenio_administration/search.html" %}
+
+  {% block javascript %}
+  {{ super() }}
+  {{ webpack['invenio-vocabularies-search.js'] }}
+{% endblock %}

invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabulary-details.html

@@ -0,0 +1,71 @@
+{#
+    Copyright (C) 2024 CERN.
+  
+    Invenio App RDM is free software; you can redistribute it and/or modify it
+    under the terms of the MIT License; see LICENSE file for more details.
+  #}
+  
+  {%- from "invenio_administration/macros.html" import go_back %}
+  
+  {% extends "invenio_administration/search.html" %}
+  
+  {% block admin_main_column %}
+    <main id="admin-main-content"
+          class="ten wide mobile twelve wide tablet thirteen wide computer fourteen wide widescreen column">
+  
+      <div class="ui container fluid pl-10 pr-10 pt-10 pb-10">
+        {{ go_back() }}
+      <div class="ui grid">
+        <div class="column six wide">
+          <h1 class="ui header">{{ title or name }}</h1>
+        </div>
+        <div class="column ten wide right aligned">
+          <a class="ui icon labeled button"
+              href="{{ edit_endpoint }}">
+            <i aria-hidden="true"
+                class="cog icon"></i>{{ _("Settings") }}
+          </a>
+          <a class="ui icon labeled button"
+              href="{{ create_ui_endpoint }}">
+            <i aria-hidden="true"
+                class="calendar icon"></i>{{ _("Schedule") }}
+          </a>
+          <a class="ui icon labeled button"
+              href="{{ run_endpoint }}">
+            <i aria-hidden="true"
+                class="play icon"></i>{{ _("Run now") }}
+          </a>
+        </div>
+      </div>
+        {% block admin_page_content %}
+          <div class="ui divider" aria-hidden="true"></div>
+  
+          {%- block search_app %}
+            <div
+              id="invenio-search-config"
+              data-title='{{ title | tojson }}'
+              data-resource-name='{{ resource_name | tojson }}'
+              data-invenio-search-config='{{ search_config() | tojson }}'
+              data-fields='{{ fields | tojson }}'
+              data-display-search='{{ display_search | tojson }}'
+              data-display-read='{{ display_read | tojson }}'
+              data-display-edit='{{ display_edit | tojson }}'
+              data-display-delete='{{ display_delete | tojson }}'
+              data-resource-schema='{{ resource_schema | tojson }}'
+              data-actions='{{ actions | tojson }}'
+              data-api-endpoint='{{ api_endpoint }}'
+              data-pid-path='{{ pid_path | tojson }}'
+              data-create-endpoint='{{ create_ui_endpoint }}'
+              data-list-endpoint='{{ list_ui_endpoint }}'
+            >
+            </div>
+          {%- endblock search_app %}
+        {% endblock admin_page_content %}
+      </div>
+    </main>
+  {% endblock %}
+  
+  {% block javascript %}
+    {{ super() }}
+    {{ webpack['invenio-jobs-details.js'] }}
+  {% endblock %}

invenio_vocabularies/views.py

@@ -44,3 +44,10 @@ def create_names_blueprint_from_app(app):
 def create_subjects_blueprint_from_app(app):
     """Create app blueprint."""
     return app.extensions["invenio-vocabularies"].subjects_resource.as_blueprint()
+
+
+def create_list_blueprint_from_app(app):
+    """Create app blueprint."""
+    return app.extensions[
+        "invenio-vocabularies"
+    ].vocabulary_admin_resource.as_blueprint()