invenio-vocabularies 4.3.0__tar.gz → 4.5.0__tar.gz

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/CHANGES.rst

@@ -8,6 +8,16 @@
 Changes
 =======
 
+Version v4.5.0 (released 2024-08-21)
+
+- ror: fix dump modified date comparison
+- subjects: mesh: add mesh subject datastreams
+- change schema of subject: add text and synonyms switch to datastreams
+
+Version v4.4.0 (released 2024-08-09)
+
+- services: use and adjust vnd.inveniordm.v1+json http accept header
+
 Version v4.3.0 (released 2024-08-05)
 
 - names: make names_exclude_regex configurable

{invenio-vocabularies-4.3.0/invenio_vocabularies.egg-info → invenio-vocabularies-4.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 4.3.0
+Version: 4.5.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,16 @@ Description: ..
         Changes
         =======
         
+        Version v4.5.0 (released 2024-08-21)
+        
+        - ror: fix dump modified date comparison
+        - subjects: mesh: add mesh subject datastreams
+        - change schema of subject: add text and synonyms switch to datastreams
+        
+        Version v4.4.0 (released 2024-08-09)
+        
+        - services: use and adjust vnd.inveniordm.v1+json http accept header
+        
         Version v4.3.0 (released 2024-08-05)
         
         - names: make names_exclude_regex configurable

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "4.3.0"
+__version__ = "4.5.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/package.json

@@ -13,7 +13,6 @@
     "coveralls": "^3.0.0",
     "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.15.0",
-    "enzyme-to-json": "^3.4.0",
     "expect": "^26.0.0",
     "lodash": "^4.17.0",
     "luxon": "^1.23.0",
@@ -22,10 +21,5 @@
     "react-scripts": "^5.0.1",
     "semantic-ui-react": "^2.1.0",
     "react-overridable": "^0.0.3"
-  },
-  "jest": {
-    "snapshotSerializers": [
-      "enzyme-to-json/serializer"
-    ]
   }
 }

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/config.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -104,7 +105,11 @@ VOCABULARIES_NAMES_SCHEMES = {
 }
 """Names allowed identifier schemes."""
 
-# configure CUSTOM_VOCABULARY_TYPES to differentiate output. Is used in VocabulariesServiceConfig
+VOCABULARIES_SUBJECTS_SCHEMES = {
+    "gnd": {"label": _("GND"), "validator": idutils.is_gnd, "datacite": "GND"},
+}
+"""Subjects allowed identifier schemes."""
+
 VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
     "names",
     "affiliations",
@@ -112,6 +117,7 @@ VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
     "funders",
     "subjects",
 ]
+"""List of custom vocabulary types."""
 
 VOCABULARIES_DATASTREAM_READERS = {
     "csv": CSVReader,

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -8,11 +8,13 @@
 
 """Vocabulary affiliations."""
 
+from flask_resources import JSONSerializer, ResponseHandler
 from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
 from invenio_records_resources.records.systemfields import ModelPIDField
+from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...services.permissions import PermissionPolicy
 from .config import AffiliationsSearchOptions, service_components
@@ -46,4 +48,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/affiliations",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -10,8 +10,8 @@
 """ROR-related Datastreams Readers/Writers/Transformers module."""
 
 import io
-from datetime import datetime
 
+import arrow
 import requests
 from idutils import normalize_ror
 
@@ -33,6 +33,26 @@ class RORHTTPReader(BaseReader):
             "RORHTTPReader downloads one file and therefore does not iterate through items"
         )
 
+    def _get_last_dump_date(self, linksets):
+        """Get the last dump date."""
+        for linkset in linksets:
+            metadata_formats = linkset.get("describedby", [])
+            for format_link in metadata_formats:
+                if format_link.get("type") == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        format_link["href"],
+                        headers={"Accept": format_link["type"]},
+                    )
+                    json_ld_reponse.raise_for_status()
+                    json_ld_data = json_ld_reponse.json()
+
+                    last_dump_date = arrow.get(json_ld_data["dateCreated"])
+                    return last_dump_date
+        else:
+            raise ReaderError(
+                "Couldn't find JSON-LD in publisher's linkset to determine last dump date."
+            )
+
     def read(self, item=None, *args, **kwargs):
         """Reads the latest ROR data dump ZIP file from Zenodo and yields an in-memory binary stream of it."""
         if item:
@@ -54,39 +74,21 @@ class RORHTTPReader(BaseReader):
             headers={"Accept": "application/linkset+json"},
         )
         linkset_response.raise_for_status()
+        linksets = linkset_response.json()["linkset"]
 
         if self._since:
-            for link in linkset_response.json()["linkset"]:
-                if "type" in link and link["type"] == "application/ld+json":
-                    json_ld_reponse = requests.get(
-                        link["anchor"], headers={"Accept": link["type"]}
-                    )
-                    json_ld_reponse.raise_for_status()
-
-                    # TODO Update to use dateCreated once the field is added to InvenioRDM. (https://github.com/inveniosoftware/invenio-rdm-records/issues/1777)
-                    last_dump_date = json_ld_reponse.json()["datePublished"]
-                    if datetime.fromisoformat(last_dump_date) < datetime.fromisoformat(
-                        self._since
-                    ):
-                        return
-                    break
-            else:
-                raise ReaderError("Couldn't find json-ld in publisher's linkset.")
-
-        # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = linkset_response.json()["linkset"][0]
-
-        # Extract the URL of the only ZIP file linked to the record.
-        landing_page_zip_items = [
-            item
-            for item in landing_page_linkset["item"]
-            if item["type"] == "application/zip"
-        ]
-        if len(landing_page_zip_items) != 1:
-            raise ReaderError(
-                f"Expected 1 ZIP item but got {len(landing_page_zip_items)}"
-            )
-        file_url = landing_page_zip_items[0]["href"]
+            last_dump_date = self._get_last_dump_date(linksets)
+            if last_dump_date < arrow.get(self._since):
+                return
+
+        for linkset in linksets:
+            items = linkset.get("item", [])
+            zip_files = [item for item in items if item["type"] == "application/zip"]
+            if len(zip_files) == 1:
+                file_url = zip_files[0]["href"]
+                break
+            if len(zip_files) > 1:
+                raise ReaderError(f"Expected 1 ZIP item but got {len(zip_files)}")
 
         # Download the ZIP file and fully load the response bytes content in memory.
         # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/names/names.py

@@ -8,6 +8,7 @@
 
 """Vocabulary names."""
 
+from flask_resources import JSONSerializer, ResponseHandler
 from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
@@ -18,6 +19,7 @@ from invenio_records_resources.records.systemfields import (
     ModelPIDField,
     PIDListRelation,
 )
+from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...services.permissions import PermissionPolicy
 from ..affiliations.api import Affiliation
@@ -63,4 +65,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/names",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/subjects/config.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,13 +10,19 @@
 
 """Subjects configuration."""
 
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
+from werkzeug.local import LocalProxy
 
 from ...services.components import PIDComponent
 from ...services.querystr import FilteredSuggestQueryParser
 
+subject_schemes = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_SUBJECTS_SCHEMES"]
+)
+
 
 class SubjectsSearchOptions(SearchOptions):
     """Search options."""
@@ -23,9 +30,8 @@ class SubjectsSearchOptions(SearchOptions):
     suggest_parser_cls = FilteredSuggestQueryParser.factory(
         filter_field="scheme",
         fields=[  # suggest fields
-            "subject^100",
-            "subject._2gram",
-            "subject._3gram",
+            "title.*^100",
+            "synonyms^20",
         ],
     )
 

invenio-vocabularies-4.5.0/invenio_vocabularies/contrib/subjects/datastreams.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 University of Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Names datastreams, transformers, writers and readers."""
+
+from invenio_access.permissions import system_identity
+from invenio_i18n import lazy_gettext as _
+
+from ...datastreams.writers import ServiceWriter
+from .mesh.datastreams import VOCABULARIES_DATASTREAM_READERS as mesh_readers
+from .mesh.datastreams import VOCABULARIES_DATASTREAM_TRANSFORMERS as mesh_transformers
+from .mesh.datastreams import VOCABULARIES_DATASTREAM_WRITERS as mesh_writers
+
+
+class SubjectsServiceWriter(ServiceWriter):
+    """Subjects Service Writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "subjects")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+VOCABULARIES_DATASTREAM_READERS = {**mesh_readers}
+"""Subjects Data Streams readers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {**mesh_transformers}
+"""Subjects Data Streams transformers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "subjects-service": SubjectsServiceWriter,
+    **mesh_writers,
+}
+"""Subjects Data Streams writers."""
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {"type": "yaml"},
+    ],
+    "writers": [
+        {
+            "type": "subjects-service",
+        }
+    ],
+}
+"""Data Stream configuration."""

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/subjects/jsonschemas/subjects/subject-v1.0.0.json

@@ -25,6 +25,18 @@
     "subject": {
       "description": "Human readable label.",
       "type": "string"
+    },
+    "title": {
+      "description": "Human readable label in different languages.",
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "synonyms": {
+      "description": "Synonyms of the subject label.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
     }
   }
 }

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/subjects/mappings/os-v1/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -56,6 +67,13 @@
           }
         }
       },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
       "tags": {
         "type": "keyword"
       }

invenio-vocabularies-4.5.0/invenio_vocabularies/contrib/subjects/mappings/os-v2/subjects/subject-v1.0.0.json

@@ -0,0 +1,82 @@
+{
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "scheme": {
+        "type": "keyword"
+      },
+      "subject": {
+        "type": "text",
+        "copy_to": "subject_sort",
+        "fields": {
+          "suggest": {
+            "type": "search_as_you_type"
+          }
+        }
+      },
+      "subject_sort": {
+        "type": "keyword"
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio-vocabularies-4.5.0/invenio_vocabularies/contrib/subjects/mappings/v7/subjects/subject-v1.0.0.json

@@ -0,0 +1,82 @@
+{
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "scheme": {
+        "type": "keyword"
+      },
+      "subject": {
+        "type": "text",
+        "copy_to": "subject_sort",
+        "fields": {
+          "suggest": {
+            "type": "search_as_you_type"
+          }
+        }
+      },
+      "subject_sort": {
+        "type": "keyword"
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio-vocabularies-4.5.0/invenio_vocabularies/contrib/subjects/mesh/datastreams.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH subjects datastreams, transformers, writers and readers."""
+
+from invenio_vocabularies.datastreams.transformers import (
+    BaseTransformer,
+    TransformerError,
+)
+
+
+class MeshSubjectsTransformer(BaseTransformer):
+    """MeSH subjects Transformer."""
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Apply transformation on steam entry."""
+        entry_data = stream_entry.entry
+
+        # ID in MeSH data is the URL, ex. https://id.nlm.nih.gov/mesh/D000001
+        # We just want to use the ID prefixed by "mesh:""
+        try:
+            mesh_id = entry_data["id"].split("/")[-1]
+        except Exception:
+            raise TransformerError("Not a valid MeSH ID.")
+
+        entry_data["id"] = "mesh:" + mesh_id
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""MeSH datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+"""MeSH subject datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"mesh-subjects": MeshSubjectsTransformer}
+"""MeSH subjects datastream transformers."""

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/subjects/schema.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021 Northwestern University.
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,10 +10,15 @@
 
 """Subjects schema."""
 
-from invenio_i18n import lazy_gettext as _
+from invenio_i18n import get_locale
+from marshmallow import fields, pre_load
 from marshmallow_utils.fields import SanitizedUnicode
 
-from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
+from ...services.schema import (
+    BaseVocabularySchema,
+    ContribVocabularyRelationSchema,
+    i18n_strings,
+)
 
 
 class SubjectSchema(BaseVocabularySchema):
@@ -24,6 +30,16 @@ class SubjectSchema(BaseVocabularySchema):
     id = SanitizedUnicode(required=True)
     scheme = SanitizedUnicode(required=True)
     subject = SanitizedUnicode(required=True)
+    title = i18n_strings
+    synonyms = fields.List(SanitizedUnicode())
+
+    @pre_load
+    def add_subject_from_title(self, data, **kwargs):
+        """Add subject from title if not present."""
+        locale = get_locale().language
+        if "subject" not in data:
+            data["subject"] = data["title"].get(locale) or data["title"].values()[0]
+        return data
 
 
 class SubjectRelationSchema(ContribVocabularyRelationSchema):
@@ -33,3 +49,5 @@ class SubjectRelationSchema(ContribVocabularyRelationSchema):
     parent_field_name = "subjects"
     subject = SanitizedUnicode()
     scheme = SanitizedUnicode()
+    title = i18n_strings
+    synonyms = fields.List(SanitizedUnicode())

{invenio-vocabularies-4.3.0 → invenio-vocabularies-4.5.0}/invenio_vocabularies/contrib/subjects/subjects.py

@@ -9,9 +9,11 @@
 
 """Vocabulary subjects."""
 
+from flask_resources import JSONSerializer, ResponseHandler
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...records.pidprovider import PIDProviderFactory
 from ...records.systemfields import BaseVocabularyPIDFieldContext
@@ -42,4 +44,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/subjects",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )