invenio-vocabularies 3.4.1__tar.gz → 4.0.0__tar.gz

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/CHANGES.rst

@@ -8,10 +8,15 @@
 Changes
 =======
 
-Version 3.4.1 (released 2024-04-19)
-
-- funders: exclude ROR schema v2 json (#309)
-
+Version 4.0.0 (released 2024-06-04)
+
+- datastreams: implement factories for generating vocabulary configurations
+- datastreams: added ROR HTTP reader
+- funders: use ROR v2 dump instead of v1
+- datastreams: added celery task for funders using ROR HTTP reader
+- datastreams: add OpenAIRE Project HTTP Reader
+- datastreams: fix OpenAIRE graph dataset parsing
+- installation: upgrade invenio-records-resources
 
 Version 3.4.0 (released 2024-04-19)
 

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 3.4.1
+Version: 4.0.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,10 +50,15 @@ Description: ..
         Changes
         =======
         
-        Version 3.4.1 (released 2024-04-19)
-        
-        - funders: exclude ROR schema v2 json (#309)
+        Version 4.0.0 (released 2024-06-04)
         
+        - datastreams: implement factories for generating vocabulary configurations
+        - datastreams: added ROR HTTP reader
+        - funders: use ROR v2 dump instead of v1
+        - datastreams: added celery task for funders using ROR HTTP reader
+        - datastreams: add OpenAIRE Project HTTP Reader
+        - datastreams: fix OpenAIRE graph dataset parsing
+        - installation: upgrade invenio-records-resources
         
         Version 3.4.0 (released 2024-04-19)
         

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "3.4.1"
+__version__ = "4.0.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/CustomAwardForm.js

@@ -28,8 +28,8 @@ function CustomAwardForm({ deserializeFunder, selectedFunding }) {
       funderPID = funderItem.id;
     }
 
-    if (funderItem.country) {
-      funderCountry = funderItem.country;
+    if (funderItem.country_name) {
+      funderCountry = funderItem.country_name;
     }
 
     if (!funderName && !funderPID) {

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/invenio_vocabularies/cli.py

@@ -9,46 +9,14 @@
 
 """Commands to create and manage vocabularies."""
 
-from copy import deepcopy
 
 import click
-import yaml
 from flask.cli import with_appcontext
 from invenio_access.permissions import system_identity
 from invenio_pidstore.errors import PIDDeletedError, PIDDoesNotExistError
-from invenio_records_resources.proxies import current_service_registry
 
-from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
-from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
-from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
 from .datastreams import DataStreamFactory
-
-
-def get_config_for_ds(vocabulary, filepath=None, origin=None):
-    """Calculates the configuration for a Data Stream."""
-    config = None
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        config = deepcopy(names_ds_config)
-    elif vocabulary == "funders":
-        config = deepcopy(funders_ds_config)
-    elif vocabulary == "awards":
-        config = deepcopy(awards_ds_config)
-
-    if config:
-        if filepath:
-            with open(filepath) as f:
-                config = yaml.safe_load(f).get(vocabulary)
-        if origin:
-            config["readers"][0].setdefault("args", {})
-            config["readers"][0]["args"]["origin"] = origin
-
-    return config
-
-
-def get_service_for_vocabulary(vocabulary):
-    """Calculates the configuration for a Data Stream."""
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        return current_service_registry.get("names")
+from .factories import get_vocabulary_config
 
 
 @click.group()
@@ -111,7 +79,9 @@ def import_vocab(vocabulary, filepath=None, origin=None, num_samples=None):
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+
     success, errored, filtered = _process_vocab(config, num_samples)
 
     _output_process(vocabulary, "imported", success, errored, filtered)
@@ -127,8 +97,8 @@ def update(vocabulary, filepath=None, origin=None):
     if not filepath and not origin:
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
-
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
 
     for w_conf in config["writers"]:
         w_conf["args"]["update"] = True
@@ -153,7 +123,8 @@ def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=Non
         )
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
     if not filepath:
         config["writers"] = [{"type": "yaml", "args": {"filepath": target}}]
 
@@ -176,8 +147,8 @@ def delete(vocabulary, identifier, all):
     if not id and not all:
         click.secho("An identifier or the --all flag must be present.", fg="red")
         exit(1)
-
-    service = get_service_for_vocabulary(vocabulary)
+    vc = get_vocabulary_config(vocabulary)
+    service = vc.get_service()
     if identifier:
         try:
             if service.delete(identifier, system_identity):

invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/awards/datastreams.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Awards datastreams, transformers, writers and readers."""
+import io
+
+import requests
+from invenio_access.permissions import system_identity
+from invenio_i18n import lazy_gettext as _
+
+from ...datastreams.errors import ReaderError, TransformerError
+from ...datastreams.readers import BaseReader
+from ...datastreams.transformers import BaseTransformer
+from ...datastreams.writers import ServiceWriter
+from .config import awards_ec_ror_id, awards_openaire_funders_mapping
+
+
+class OpenAIREProjectHTTPReader(BaseReader):
+    """OpenAIRE Project HTTP Reader returning an in-memory binary stream of the latest OpenAIRE Graph Dataset project tar file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "OpenAIREProjectHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest OpenAIRE Graph Dataset project tar file from Zenodo and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "OpenAIREProjectHTTPReader does not support being chained after another reader"
+            )
+
+        if self._origin == "full":
+            # OpenAIRE Graph Dataset
+            api_url = "https://zenodo.org/api/records/3516917"
+        elif self._origin == "diff":
+            # OpenAIRE Graph dataset: new collected projects
+            api_url = "https://zenodo.org/api/records/6419021"
+        else:
+            raise ReaderError("The --origin option should be either 'full' or 'diff'")
+
+        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the OpenAIRE Graph Dataset.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        headers = {"Accept": "application/linkset+json"}
+        api_resp = requests.get(api_url, headers=headers)
+        api_resp.raise_for_status()
+
+        # Extract the Landing page Link Set Object located as the first (index 0) item.
+        landing_page_linkset = api_resp.json()["linkset"][0]
+
+        # Extract the URL of the only project tar file linked to the record.
+        landing_page_project_tar_items = [
+            item
+            for item in landing_page_linkset["item"]
+            if item["type"] == "application/x-tar"
+            and item["href"].endswith("/project.tar")
+        ]
+        if len(landing_page_project_tar_items) != 1:
+            raise ReaderError(
+                f"Expected 1 project tar item but got {len(landing_page_project_tar_items)}"
+            )
+        file_url = landing_page_project_tar_items[0]["href"]
+
+        # Download the project tar file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `tarfile.open`).
+        # Using directly `file_resp.raw` is not possible since `tarfile.open` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+class AwardsServiceWriter(ServiceWriter):
+    """Funders service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "awards")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+class OpenAIREProjectTransformer(BaseTransformer):
+    """Transforms an OpenAIRE project record into an award record."""
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+        award = {}
+
+        code = record["code"]
+
+        # The `id` should follow the format `sourcePrefix::md5(localId)` where `sourcePrefix` is 12 characters long.
+        # See: https://graph.openaire.eu/docs/data-model/pids-and-identifiers#identifiers-in-the-graph
+        #
+        # The format of `id` in the full OpenAIRE Graph Dataset (https://doi.org/10.5281/zenodo.3516917)
+        # follows this format (e.g. 'abc_________::0123456789abcdef0123456789abcdef').
+        # However, the format of `id` in the new collected projects dataset (https://doi.org/10.5281/zenodo.6419021)
+        # does not follow this format, and has a `40|` prefix (e.g. '40|abc_________::0123456789abcdef0123456789abcdef').
+        #
+        # The number '40' corresponds to the entity types 'Project'.
+        # See: https://ec.europa.eu/research/participants/documents/downloadPublic?documentIds=080166e5a3a1a213&appId=PPGMS
+        # See: https://graph.openaire.eu/docs/5.0.0/data-model/entities/project#id
+        openaire_funder_prefix = record["id"].split("::", 1)[0].split("|", 1)[-1]
+
+        funder_id = awards_openaire_funders_mapping.get(openaire_funder_prefix)
+        if funder_id is None:
+            raise TransformerError(
+                _(
+                    "Unknown OpenAIRE funder prefix {openaire_funder_prefix}".format(
+                        openaire_funder_prefix=openaire_funder_prefix
+                    )
+                )
+            )
+
+        award["id"] = f"{funder_id}::{code}"
+
+        funding = next(iter(record.get("funding", [])), None)
+        if funding:
+            funding_stream_id = funding.get("funding_stream", {}).get("id", "")
+            # Example funding stream ID: `EC::HE::HORIZON-AG-UN`. We need the `EC`
+            # string, i.e. the second "part" of the identifier.
+            program = next(iter(funding_stream_id.split("::")[1:2]), "")
+            if program:
+                award["program"] = program
+
+        identifiers = []
+        if funder_id == awards_ec_ror_id:
+            identifiers.append(
+                {
+                    "identifier": f"https://cordis.europa.eu/projects/{code}",
+                    "scheme": "url",
+                }
+            )
+        elif record.get("websiteurl"):
+            identifiers.append(
+                {"identifier": record.get("websiteurl"), "scheme": "url"}
+            )
+
+        if identifiers:
+            award["identifiers"] = identifiers
+
+        award["number"] = code
+
+        # `title` is a mandatory attribute of the `Project` object in the OpenAIRE Graph Data Model.
+        # See: https://graph.openaire.eu/docs/data-model/entities/project#title
+        # However, 15'000+ awards for the FCT funder (and 1 record the NIH funder) are missing a title attribute.
+        if "title" not in record:
+            raise TransformerError(
+                _(
+                    "Missing title attribute for award {award_id}".format(
+                        award_id=award["id"]
+                    )
+                )
+            )
+        award["title"] = {"en": record["title"]}
+
+        award["funder"] = {"id": funder_id}
+        acronym = record.get("acronym")
+        if acronym:
+            award["acronym"] = acronym
+
+        stream_entry.entry = award
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "openaire-project-http": OpenAIREProjectHTTPReader,
+}
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "openaire-award": OpenAIREProjectTransformer,
+}
+"""ORCiD Data Streams transformers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "awards-service": AwardsServiceWriter,
+}
+"""ORCiD Data Streams transformers."""
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "tar",
+            "args": {
+                "regex": "\\.json.gz$",
+                "mode": "r",
+            },
+        },
+        {"type": "gzip"},
+        {"type": "jsonl"},
+    ],
+    "transformers": [
+        {"type": "openaire-award"},
+    ],
+    "writers": [
+        {
+            "type": "awards-service",
+            "args": {
+                "identity": system_identity,
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""

invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/common/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabularies common module."""

invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/common/ror/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related module."""

invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related Datastreams Readers/Writers/Transformers module."""
+
+import io
+
+import requests
+
+from invenio_vocabularies.datastreams.errors import ReaderError
+from invenio_vocabularies.datastreams.readers import BaseReader
+
+
+class RORHTTPReader(BaseReader):
+    """ROR HTTP Reader returning an in-memory binary stream of the latest ROR data dump ZIP file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "RORHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest ROR data dump ZIP file from Zenodo and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "RORHTTPReader does not support being chained after another reader"
+            )
+
+        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the ROR data dump.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        headers = {"Accept": "application/linkset+json"}
+        api_url = "https://zenodo.org/api/records/6347574"
+        api_resp = requests.get(api_url, headers=headers)
+        api_resp.raise_for_status()
+
+        # Extract the Landing page Link Set Object located as the first (index 0) item.
+        landing_page_linkset = api_resp.json()["linkset"][0]
+
+        # Extract the URL of the only ZIP file linked to the record.
+        landing_page_zip_items = [
+            item
+            for item in landing_page_linkset["item"]
+            if item["type"] == "application/zip"
+        ]
+        if len(landing_page_zip_items) != 1:
+            raise ReaderError(
+                f"Expected 1 ZIP item but got {len(landing_page_zip_items)}"
+            )
+        file_url = landing_page_zip_items[0]["href"]
+
+        # Download the ZIP file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).
+        # Using directly `file_resp.raw` is not possible since `zipfile.ZipFile` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "ror-http": RORHTTPReader,
+}

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/invenio_vocabularies/contrib/funders/config.py

@@ -31,6 +31,8 @@ class FundersSearchOptions(SearchOptions):
         fields=[
             "name^100",
             "identifiers.identifier^10",
+            "acronym^10",
+            "aliases^10",
         ]
     )
 

{invenio-vocabularies-3.4.1 → invenio-vocabularies-4.0.0}/invenio_vocabularies/contrib/funders/datastreams.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -38,36 +39,70 @@ class RORTransformer(BaseTransformer):
         """Applies the transformation to the stream entry."""
         record = stream_entry.entry
         funder = {}
+        funder["title"] = {}
 
         funder["id"] = normalize_ror(record.get("id"))
         if not funder["id"]:
             raise TransformerError(_("Id not found in ROR entry."))
 
-        funder["name"] = record.get("name")
+        aliases = []
+        acronym = None
+        for name in record.get("names"):
+            # Some name entries have a `lang` key with a `None` value.
+            # Therefore, providing a default value to `name.get("lang")` is not enough,
+            # and we need instead to check if the result of `get` is None.
+            lang = name.get("lang")
+            if lang is None:
+                lang = "en"
+
+            if "ror_display" in name["types"]:
+                funder["name"] = name["value"]
+            if "label" in name["types"]:
+                funder["title"][lang] = name["value"]
+            if "alias" in name["types"]:
+                aliases.append(name["value"])
+            if "acronym" in name["types"]:
+                # The first acronyn goes in acronym field to maintain
+                # compatability with existing data structure
+                if not acronym:
+                    acronym = name["value"]
+                else:
+                    aliases.append(name["value"])
+        if acronym:
+            funder["acronym"] = acronym
+        if aliases:
+            funder["aliases"] = aliases
+
+        # ror_display is required and should be in every entry
         if not funder["name"]:
-            raise TransformerError(_("Name not found in ROR entry."))
+            raise TransformerError(
+                _("Name with type ror_display not found in ROR entry.")
+            )
 
-        country_code = record.get("country", {}).get("country_code")
-        if country_code:
-            funder["country"] = country_code
+        # This only gets the first location, to maintain compatability
+        # with existing data structure
+        location = record.get("locations", [{}])[0].get("geonames_details", {})
+        funder["country"] = location.get("country_code")
+        funder["country_name"] = location.get("country_name")
+        funder["location_name"] = location.get("name")
 
-        funder["title"] = {"en": funder["name"]}
-        for label in record.get("labels", []):
-            funder["title"][label["iso639"]] = label["label"]
+        funder["types"] = record.get("types")
+
+        status = record.get("status")
+        funder["status"] = status
 
         # The ROR is always listed in identifiers, expected by serialization
         funder["identifiers"] = [{"identifier": funder["id"], "scheme": "ror"}]
         valid_schemes = set(funder_schemes.keys())
         fund_ref = "fundref"
         valid_schemes.add(fund_ref)
-        for scheme, identifier in record.get("external_ids", {}).items():
-            scheme = scheme.lower()
+        for identifier in record.get("external_ids"):
+            scheme = identifier["type"]
             if scheme in valid_schemes:
                 value = identifier.get("preferred") or identifier.get("all")[0]
                 if scheme == fund_ref:
                     value = f"{funder_fundref_doi_prefix}/{value}"
                     scheme = "doi"
-
                 funder["identifiers"].append(
                     {
                         "identifier": value,
@@ -96,7 +131,7 @@ DATASTREAM_CONFIG = {
         {
             "type": "zip",
             "args": {
-                "regex": "(?<!_schema_v2)\\.json$",
+                "regex": "_schema_v2\\.json$",
             },
         },
         {"type": "json"},

invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json

@@ -0,0 +1,62 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "local://funders/funder-v1.0.0.json",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "$schema": {
+      "$ref": "local://definitions-v1.0.0.json#/$schema"
+    },
+    "country": {
+      "type": "string",
+      "description": "Represents a funder's origin country as a country code."
+    },
+    "country_name": {
+      "type": "string",
+      "description": "Represents a funder's origin country as a full name."
+    },
+    "location_name": {
+      "type": "string",
+      "description": "Represents a funder's location name (usually a city)."
+    },
+    "identifiers": {
+      "description": "Alternate identifiers for the record.",
+      "type": "array",
+      "items": {
+        "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "name": {
+      "type": "string",
+      "description": "Funders name."
+    },
+    "acronym": {
+      "type": "string",
+      "description": "Acronym for funders name."
+    },
+    "status": {
+        "type": "string",
+        "description": "Status of the funder."
+    },
+    "aliases": {
+      "description": "Alternate names for the funder.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "types": {
+      "description": "Types of funders.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "title": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    }
+  }
+}

{invenio-vocabularies-3.4.1/invenio_vocabularies/contrib/funders/mappings/os-v2 → invenio-vocabularies-4.0.0/invenio_vocabularies/contrib/funders/mappings/os-v1}/funders/funder-v1.0.0.json

@@ -57,6 +57,24 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },