invenio-vocabularies 4.1.1__tar.gz → 4.2.0__tar.gz

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/CHANGES.rst

@@ -8,6 +8,14 @@
 Changes
 =======
 
+Version v4.2.0 (released 2024-07-24)
+
+- ror: check last update; use ld+json for metadata (#367)
+- tasks: remove import funders task
+- funders: add and export custom transformer
+- affiliations: add and export custom transformer
+- datastreams: implement asynchronous writer
+
 Version v4.1.1 (released 2024-07-15)
 
 - installation: use invenio-oaipmh-scythe from PyPI

{invenio-vocabularies-4.1.1/invenio_vocabularies.egg-info → invenio-vocabularies-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 4.1.1
+Version: 4.2.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,14 @@ Description: ..
         Changes
         =======
         
+        Version v4.2.0 (released 2024-07-24)
+        
+        - ror: check last update; use ld+json for metadata (#367)
+        - tasks: remove import funders task
+        - funders: add and export custom transformer
+        - affiliations: add and export custom transformer
+        - datastreams: implement asynchronous writer
+        
         Version v4.1.1 (released 2024-07-15)
         
         - installation: use invenio-oaipmh-scythe from PyPI

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "4.1.1"
+__version__ = "4.2.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/administration/views/vocabularies.py

@@ -8,6 +8,7 @@
 # details.
 
 """Vocabularies admin interface."""
+
 from invenio_administration.views.base import (
     AdminResourceEditView,
     AdminResourceListView,

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/cli.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -9,7 +9,6 @@
 
 """Commands to create and manage vocabularies."""
 
-
 import click
 from flask.cli import with_appcontext
 from invenio_access.permissions import system_identity
@@ -101,7 +100,10 @@ def update(vocabulary, filepath=None, origin=None):
     config = vc.get_config(filepath, origin)
 
     for w_conf in config["writers"]:
-        w_conf["args"]["update"] = True
+        if w_conf["type"] == "async":
+            w_conf["args"]["writer"]["args"]["update"] = True
+        else:
+            w_conf["args"]["update"] = True
 
     success, errored, filtered = _process_vocab(config)
 

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/config.py

@@ -24,7 +24,7 @@ from .datastreams.readers import (
     ZipReader,
 )
 from .datastreams.transformers import XMLTransformer
-from .datastreams.writers import ServiceWriter, YamlWriter
+from .datastreams.writers import AsyncWriter, ServiceWriter, YamlWriter
 from .resources import VocabulariesResourceConfig
 from .services.config import VocabulariesServiceConfig
 
@@ -134,6 +134,7 @@ VOCABULARIES_DATASTREAM_TRANSFORMERS = {
 VOCABULARIES_DATASTREAM_WRITERS = {
     "service": ServiceWriter,
     "yaml": YamlWriter,
+    "async": AsyncWriter,
 }
 """Data Streams writers."""
 

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/affiliations/api.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 Affiliation = record_type.record_cls

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/affiliations/datastreams.py

@@ -9,11 +9,11 @@
 
 """Affiliations datastreams, transformers, writers and readers."""
 
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
 from ...datastreams.writers import ServiceWriter
-from .config import affiliation_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class AffiliationsServiceWriter(ServiceWriter):
@@ -29,9 +29,35 @@ class AffiliationsServiceWriter(ServiceWriter):
         return entry["id"]
 
 
+class AffiliationsRORTransformer(RORTransformer):
+    """Affiliations ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_AFFILIATION_SCHEMES")
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Affiliations datastream readers."""
+
 VOCABULARIES_DATASTREAM_WRITERS = {
     "affiliations-service": AffiliationsServiceWriter,
 }
+"""Affiliations datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-affiliations": AffiliationsRORTransformer,
+}
+"""Affiliations datastream transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -46,17 +72,16 @@ DATASTREAM_CONFIG = {
     ],
     "transformers": [
         {
-            "type": "ror",
-            "args": {
-                "vocab_schemes": affiliation_schemes,
-            },
+            "type": "ror-affiliations",
         },
     ],
     "writers": [
         {
-            "type": "affiliations-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "affiliations-service",
+                }
             },
         }
     ],

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/affiliations/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 AffiliationsServiceConfig = record_type.service_config_cls

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/awards/awards.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary awards."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/awards/datastreams.py

@@ -7,6 +7,7 @@
 # details.
 
 """Awards datastreams, transformers, writers and readers."""
+
 import io
 
 import requests

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/awards/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary awards."""
 
-
 from .awards import record_type
 
 AwardsServiceConfig = record_type.service_config_cls

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -10,6 +10,7 @@
 """ROR-related Datastreams Readers/Writers/Transformers module."""
 
 import io
+from datetime import datetime
 
 import requests
 from idutils import normalize_ror
@@ -22,6 +23,11 @@ from invenio_vocabularies.datastreams.transformers import BaseTransformer
 class RORHTTPReader(BaseReader):
     """ROR HTTP Reader returning an in-memory binary stream of the latest ROR data dump ZIP file."""
 
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor."""
+        self._since = since
+        super().__init__(origin, mode, *args, **kwargs)
+
     def _iter(self, fp, *args, **kwargs):
         raise NotImplementedError(
             "RORHTTPReader downloads one file and therefore does not iterate through items"
@@ -34,15 +40,41 @@ class RORHTTPReader(BaseReader):
                 "RORHTTPReader does not support being chained after another reader"
             )
 
+        # Follow the DOI to get the link of the linkset
+        dataset_doi_link = "https://doi.org/10.5281/zenodo.6347574"
+        landing_page = requests.get(dataset_doi_link, allow_redirects=True)
+        landing_page.raise_for_status()
+
         # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the ROR data dump.
         # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
-        headers = {"Accept": "application/linkset+json"}
-        api_url = "https://zenodo.org/api/records/6347574"
-        api_resp = requests.get(api_url, headers=headers)
-        api_resp.raise_for_status()
+        if "linkset" not in landing_page.links:
+            raise ReaderError("Linkset not found in the ROR dataset record.")
+        linkset_response = requests.get(
+            landing_page.links["linkset"]["url"],
+            headers={"Accept": "application/linkset+json"},
+        )
+        linkset_response.raise_for_status()
+
+        if self._since:
+            for link in linkset_response.json()["linkset"]:
+                if "type" in link and link["type"] == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        link["anchor"], headers={"Accept": link["type"]}
+                    )
+                    json_ld_reponse.raise_for_status()
+
+                    # TODO Update to use dateCreated once the field is added to InvenioRDM. (https://github.com/inveniosoftware/invenio-rdm-records/issues/1777)
+                    last_dump_date = json_ld_reponse.json()["datePublished"]
+                    if datetime.fromisoformat(last_dump_date) < datetime.fromisoformat(
+                        self._since
+                    ):
+                        return
+                    break
+            else:
+                raise ReaderError("Couldn't find json-ld in publisher's linkset.")
 
         # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = api_resp.json()["linkset"][0]
+        landing_page_linkset = linkset_response.json()["linkset"][0]
 
         # Extract the URL of the only ZIP file linked to the record.
         landing_page_zip_items = [
@@ -164,3 +196,5 @@ class RORTransformer(BaseTransformer):
 VOCABULARIES_DATASTREAM_TRANSFORMERS = {
     "ror": RORTransformer,
 }
+
+VOCABULARIES_DATASTREAM_WRITERS = {}

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/funders/datastreams.py

@@ -9,12 +9,11 @@
 
 """Funders datastreams, transformers, writers and readers."""
 
-from idutils import normalize_ror
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
 from ...datastreams.writers import ServiceWriter
-from .config import funder_fundref_doi_prefix, funder_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class FundersServiceWriter(ServiceWriter):
@@ -30,10 +29,40 @@ class FundersServiceWriter(ServiceWriter):
         return entry["id"]
 
 
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
+            )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
+
 VOCABULARIES_DATASTREAM_WRITERS = {
     "funders-service": FundersServiceWriter,
 }
-"""Funders Data Streams transformers."""
+"""Funders datastreams writers."""
+
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
+}
+"""Funders datastreams transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -48,18 +77,16 @@ DATASTREAM_CONFIG = {
     ],
     "transformers": [
         {
-            "type": "ror",
-            "args": {
-                "vocab_schemes": funder_schemes,
-                "funder_fundref_doi_prefix": funder_fundref_doi_prefix,
-            },
+            "type": "ror-funders",
         },
     ],
     "writers": [
         {
-            "type": "funders-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "funders-service",
+                }
             },
         }
     ],

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/contrib/funders/funders.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary funders."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/datastreams/readers.py

@@ -270,19 +270,38 @@ class OAIPMHReader(BaseReader):
                     self.xml.find(f".//{self._oai_namespace}metadata").getchildren()[0],
                 )
 
-        scythe.class_mapping["ListRecords"] = OAIRecord
-        try:
-            records = scythe.list_records(
-                from_=self._from,
-                until=self._until,
-                metadata_prefix=self._metadata_prefix,
-                set_=self._set,
-                ignore_deleted=True,
-            )
-            for record in records:
-                yield {"record": record}
-        except oaipmh_scythe.NoRecordsMatch:
-            raise ReaderError(f"No records found in OAI-PMH request.")
+        if self._verb == "ListRecords":
+            scythe.class_mapping["ListRecords"] = OAIRecord
+            try:
+                records = scythe.list_records(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for record in records:
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+        else:
+            scythe.class_mapping["GetRecord"] = OAIRecord
+            try:
+                headers = scythe.list_identifiers(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for header in headers:
+                    record = scythe.get_record(
+                        identifier=header.identifier,
+                        metadata_prefix=self._metadata_prefix,
+                    )
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
 
     def read(self, item=None, *args, **kwargs):
         """Reads from item or opens the file descriptor from origin."""

invenio-vocabularies-4.2.0/invenio_vocabularies/datastreams/tasks.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Data Streams Celery tasks."""
+
+from celery import shared_task
+
+from ..datastreams import StreamEntry
+from ..datastreams.factories import WriterFactory
+
+
+@shared_task(ignore_result=True)
+def write_entry(writer, entry):
+    """Write an entry.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: dictionary, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer)
+    writer.write(StreamEntry(entry))

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/datastreams/writers.py

@@ -20,6 +20,7 @@ from marshmallow import ValidationError
 
 from .datastreams import StreamEntry
 from .errors import WriterError
+from .tasks import write_entry
 
 
 class BaseWriter(ABC):
@@ -106,3 +107,21 @@ class YamlWriter(BaseWriter):
             yaml.safe_dump([stream_entry.entry], file, allow_unicode=True)
 
         return stream_entry
+
+
+class AsyncWriter(BaseWriter):
+    """Writes the entries asynchronously (celery task)."""
+
+    def __init__(self, writer, *args, **kwargs):
+        """Constructor.
+
+        :param writer: writer to use.
+        """
+        self._writer = writer
+        super().__init__(*args, **kwargs)
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_entry.delay(self._writer, stream_entry.entry)
+
+        return stream_entry

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/factories.py

@@ -7,6 +7,7 @@
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 """Generate Vocabulary Config."""
+
 from copy import deepcopy
 
 import yaml

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/records/models.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -79,9 +79,7 @@ class VocabularyScheme(db.Model):
         """Create a new vocabulary subtype."""
         banned = [",", ":"]
         for b in banned:
-            assert (
-                b not in data["id"]
-            ), f"No '{b}' allowed in VocabularyScheme.id"  # noqa
+            assert b not in data["id"], f"No '{b}' allowed in VocabularyScheme.id"
 
         with db.session.begin_nested():
             obj = cls(**data)

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/records/pidprovider.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Persistent identifier provider for vocabularies."""
 
-
 from invenio_pidstore.models import PIDStatus
 from invenio_pidstore.providers.base import BaseProvider
 

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/resources/__init__.py

@@ -7,6 +7,7 @@
 # details.
 
 """Resources module."""
+
 from invenio_vocabularies.resources.schema import L10NString, VocabularyL10Schema
 
 from .config import VocabulariesResourceConfig, VocabularyTypeResourceConfig

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/resources/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -8,6 +8,7 @@
 # details.
 
 """Vocabulary resource schema."""
+
 from marshmallow import Schema, fields
 
 from invenio_vocabularies.resources.serializer import L10NString

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/services/custom_fields/subject.py

@@ -7,6 +7,7 @@
 
 
 """Custom fields."""
+
 from invenio_i18n import lazy_gettext as _
 
 from ...contrib.subjects.api import Subject
@@ -25,7 +26,7 @@ class SubjectCF(VocabularyCF):
             vocabulary_id="subjects",
             schema=SubjectRelationSchema,
             ui_schema=SubjectRelationSchema,
-            **kwargs
+            **kwargs,
         )
         self.pid_field = Subject.pid
 

{invenio-vocabularies-4.1.1 → invenio-vocabularies-4.2.0}/invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -39,7 +39,7 @@ class VocabularyCF(BaseCF):
         sort_by=None,
         schema=VocabularyRelationSchema,
         ui_schema=VocabularyL10NItemSchema,
-        **kwargs
+        **kwargs,
     ):
         """Constructor."""
         super().__init__(name, **kwargs)

invenio-vocabularies-4.2.0/invenio_vocabularies/services/tasks.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Celery tasks."""
+
+from celery import shared_task
+from flask import current_app
+
+from ..datastreams.factories import DataStreamFactory
+
+
+@shared_task(ignore_result=True)
+def process_datastream(config):
+    """Process a datastream from config."""
+    ds = DataStreamFactory.create(
+        readers_config=config["readers"],
+        transformers_config=config.get("transformers"),
+        writers_config=config["writers"],
+    )
+
+    for result in ds.process():
+        if result.errors:
+            for err in result.errors:
+                current_app.logger.error(err)