invenio-vocabularies 3.4.1__py2.py3-none-any.whl → 4.0.0__py2.py3-none-any.whl

invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "3.4.1"
+__version__ = "4.0.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/CustomAwardForm.js

@@ -28,8 +28,8 @@ function CustomAwardForm({ deserializeFunder, selectedFunding }) {
       funderPID = funderItem.id;
     }
 
-    if (funderItem.country) {
-      funderCountry = funderItem.country;
+    if (funderItem.country_name) {
+      funderCountry = funderItem.country_name;
     }
 
     if (!funderName && !funderPID) {

invenio_vocabularies/cli.py

@@ -9,46 +9,14 @@
 
 """Commands to create and manage vocabularies."""
 
-from copy import deepcopy
 
 import click
-import yaml
 from flask.cli import with_appcontext
 from invenio_access.permissions import system_identity
 from invenio_pidstore.errors import PIDDeletedError, PIDDoesNotExistError
-from invenio_records_resources.proxies import current_service_registry
 
-from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
-from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
-from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
 from .datastreams import DataStreamFactory
-
-
-def get_config_for_ds(vocabulary, filepath=None, origin=None):
-    """Calculates the configuration for a Data Stream."""
-    config = None
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        config = deepcopy(names_ds_config)
-    elif vocabulary == "funders":
-        config = deepcopy(funders_ds_config)
-    elif vocabulary == "awards":
-        config = deepcopy(awards_ds_config)
-
-    if config:
-        if filepath:
-            with open(filepath) as f:
-                config = yaml.safe_load(f).get(vocabulary)
-        if origin:
-            config["readers"][0].setdefault("args", {})
-            config["readers"][0]["args"]["origin"] = origin
-
-    return config
-
-
-def get_service_for_vocabulary(vocabulary):
-    """Calculates the configuration for a Data Stream."""
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        return current_service_registry.get("names")
+from .factories import get_vocabulary_config
 
 
 @click.group()
@@ -111,7 +79,9 @@ def import_vocab(vocabulary, filepath=None, origin=None, num_samples=None):
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+
     success, errored, filtered = _process_vocab(config, num_samples)
 
     _output_process(vocabulary, "imported", success, errored, filtered)
@@ -127,8 +97,8 @@ def update(vocabulary, filepath=None, origin=None):
     if not filepath and not origin:
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
-
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
 
     for w_conf in config["writers"]:
         w_conf["args"]["update"] = True
@@ -153,7 +123,8 @@ def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=Non
         )
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
     if not filepath:
         config["writers"] = [{"type": "yaml", "args": {"filepath": target}}]
 
@@ -176,8 +147,8 @@ def delete(vocabulary, identifier, all):
     if not id and not all:
         click.secho("An identifier or the --all flag must be present.", fg="red")
         exit(1)
-
-    service = get_service_for_vocabulary(vocabulary)
+    vc = get_vocabulary_config(vocabulary)
+    service = vc.get_service()
     if identifier:
         try:
             if service.delete(identifier, system_identity):

invenio_vocabularies/contrib/awards/datastreams.py

@@ -1,22 +1,79 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Awards datastreams, transformers, writers and readers."""
+import io
 
+import requests
 from invenio_access.permissions import system_identity
 from invenio_i18n import lazy_gettext as _
 
-from ...datastreams.errors import TransformerError
+from ...datastreams.errors import ReaderError, TransformerError
+from ...datastreams.readers import BaseReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
 from .config import awards_ec_ror_id, awards_openaire_funders_mapping
 
 
+class OpenAIREProjectHTTPReader(BaseReader):
+    """OpenAIRE Project HTTP Reader returning an in-memory binary stream of the latest OpenAIRE Graph Dataset project tar file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "OpenAIREProjectHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest OpenAIRE Graph Dataset project tar file from Zenodo and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "OpenAIREProjectHTTPReader does not support being chained after another reader"
+            )
+
+        if self._origin == "full":
+            # OpenAIRE Graph Dataset
+            api_url = "https://zenodo.org/api/records/3516917"
+        elif self._origin == "diff":
+            # OpenAIRE Graph dataset: new collected projects
+            api_url = "https://zenodo.org/api/records/6419021"
+        else:
+            raise ReaderError("The --origin option should be either 'full' or 'diff'")
+
+        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the OpenAIRE Graph Dataset.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        headers = {"Accept": "application/linkset+json"}
+        api_resp = requests.get(api_url, headers=headers)
+        api_resp.raise_for_status()
+
+        # Extract the Landing page Link Set Object located as the first (index 0) item.
+        landing_page_linkset = api_resp.json()["linkset"][0]
+
+        # Extract the URL of the only project tar file linked to the record.
+        landing_page_project_tar_items = [
+            item
+            for item in landing_page_linkset["item"]
+            if item["type"] == "application/x-tar"
+            and item["href"].endswith("/project.tar")
+        ]
+        if len(landing_page_project_tar_items) != 1:
+            raise ReaderError(
+                f"Expected 1 project tar item but got {len(landing_page_project_tar_items)}"
+            )
+        file_url = landing_page_project_tar_items[0]["href"]
+
+        # Download the project tar file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `tarfile.open`).
+        # Using directly `file_resp.raw` is not possible since `tarfile.open` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
 class AwardsServiceWriter(ServiceWriter):
     """Funders service writer."""
 
@@ -39,7 +96,20 @@ class OpenAIREProjectTransformer(BaseTransformer):
         award = {}
 
         code = record["code"]
-        openaire_funder_prefix = record["id"].split("::")[0].split("|")[1]
+
+        # The `id` should follow the format `sourcePrefix::md5(localId)` where `sourcePrefix` is 12 characters long.
+        # See: https://graph.openaire.eu/docs/data-model/pids-and-identifiers#identifiers-in-the-graph
+        #
+        # The format of `id` in the full OpenAIRE Graph Dataset (https://doi.org/10.5281/zenodo.3516917)
+        # follows this format (e.g. 'abc_________::0123456789abcdef0123456789abcdef').
+        # However, the format of `id` in the new collected projects dataset (https://doi.org/10.5281/zenodo.6419021)
+        # does not follow this format, and has a `40|` prefix (e.g. '40|abc_________::0123456789abcdef0123456789abcdef').
+        #
+        # The number '40' corresponds to the entity types 'Project'.
+        # See: https://ec.europa.eu/research/participants/documents/downloadPublic?documentIds=080166e5a3a1a213&appId=PPGMS
+        # See: https://graph.openaire.eu/docs/5.0.0/data-model/entities/project#id
+        openaire_funder_prefix = record["id"].split("::", 1)[0].split("|", 1)[-1]
+
         funder_id = awards_openaire_funders_mapping.get(openaire_funder_prefix)
         if funder_id is None:
             raise TransformerError(
@@ -78,7 +148,20 @@ class OpenAIREProjectTransformer(BaseTransformer):
             award["identifiers"] = identifiers
 
         award["number"] = code
+
+        # `title` is a mandatory attribute of the `Project` object in the OpenAIRE Graph Data Model.
+        # See: https://graph.openaire.eu/docs/data-model/entities/project#title
+        # However, 15'000+ awards for the FCT funder (and 1 record the NIH funder) are missing a title attribute.
+        if "title" not in record:
+            raise TransformerError(
+                _(
+                    "Missing title attribute for award {award_id}".format(
+                        award_id=award["id"]
+                    )
+                )
+            )
         award["title"] = {"en": record["title"]}
+
         award["funder"] = {"id": funder_id}
         acronym = record.get("acronym")
         if acronym:
@@ -88,6 +171,10 @@ class OpenAIREProjectTransformer(BaseTransformer):
         return stream_entry
 
 
+VOCABULARIES_DATASTREAM_READERS = {
+    "openaire-project-http": OpenAIREProjectHTTPReader,
+}
+
 VOCABULARIES_DATASTREAM_TRANSFORMERS = {
     "openaire-award": OpenAIREProjectTransformer,
 }

invenio_vocabularies/contrib/common/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabularies common module."""

invenio_vocabularies/contrib/common/ror/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related module."""

invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related Datastreams Readers/Writers/Transformers module."""
+
+import io
+
+import requests
+
+from invenio_vocabularies.datastreams.errors import ReaderError
+from invenio_vocabularies.datastreams.readers import BaseReader
+
+
+class RORHTTPReader(BaseReader):
+    """ROR HTTP Reader returning an in-memory binary stream of the latest ROR data dump ZIP file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "RORHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest ROR data dump ZIP file from Zenodo and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "RORHTTPReader does not support being chained after another reader"
+            )
+
+        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the ROR data dump.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        headers = {"Accept": "application/linkset+json"}
+        api_url = "https://zenodo.org/api/records/6347574"
+        api_resp = requests.get(api_url, headers=headers)
+        api_resp.raise_for_status()
+
+        # Extract the Landing page Link Set Object located as the first (index 0) item.
+        landing_page_linkset = api_resp.json()["linkset"][0]
+
+        # Extract the URL of the only ZIP file linked to the record.
+        landing_page_zip_items = [
+            item
+            for item in landing_page_linkset["item"]
+            if item["type"] == "application/zip"
+        ]
+        if len(landing_page_zip_items) != 1:
+            raise ReaderError(
+                f"Expected 1 ZIP item but got {len(landing_page_zip_items)}"
+            )
+        file_url = landing_page_zip_items[0]["href"]
+
+        # Download the ZIP file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).
+        # Using directly `file_resp.raw` is not possible since `zipfile.ZipFile` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "ror-http": RORHTTPReader,
+}

invenio_vocabularies/contrib/funders/config.py

@@ -31,6 +31,8 @@ class FundersSearchOptions(SearchOptions):
         fields=[
             "name^100",
             "identifiers.identifier^10",
+            "acronym^10",
+            "aliases^10",
         ]
     )
 

invenio_vocabularies/contrib/funders/datastreams.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -38,36 +39,70 @@ class RORTransformer(BaseTransformer):
         """Applies the transformation to the stream entry."""
         record = stream_entry.entry
         funder = {}
+        funder["title"] = {}
 
         funder["id"] = normalize_ror(record.get("id"))
         if not funder["id"]:
             raise TransformerError(_("Id not found in ROR entry."))
 
-        funder["name"] = record.get("name")
+        aliases = []
+        acronym = None
+        for name in record.get("names"):
+            # Some name entries have a `lang` key with a `None` value.
+            # Therefore, providing a default value to `name.get("lang")` is not enough,
+            # and we need instead to check if the result of `get` is None.
+            lang = name.get("lang")
+            if lang is None:
+                lang = "en"
+
+            if "ror_display" in name["types"]:
+                funder["name"] = name["value"]
+            if "label" in name["types"]:
+                funder["title"][lang] = name["value"]
+            if "alias" in name["types"]:
+                aliases.append(name["value"])
+            if "acronym" in name["types"]:
+                # The first acronyn goes in acronym field to maintain
+                # compatability with existing data structure
+                if not acronym:
+                    acronym = name["value"]
+                else:
+                    aliases.append(name["value"])
+        if acronym:
+            funder["acronym"] = acronym
+        if aliases:
+            funder["aliases"] = aliases
+
+        # ror_display is required and should be in every entry
         if not funder["name"]:
-            raise TransformerError(_("Name not found in ROR entry."))
+            raise TransformerError(
+                _("Name with type ror_display not found in ROR entry.")
+            )
 
-        country_code = record.get("country", {}).get("country_code")
-        if country_code:
-            funder["country"] = country_code
+        # This only gets the first location, to maintain compatability
+        # with existing data structure
+        location = record.get("locations", [{}])[0].get("geonames_details", {})
+        funder["country"] = location.get("country_code")
+        funder["country_name"] = location.get("country_name")
+        funder["location_name"] = location.get("name")
 
-        funder["title"] = {"en": funder["name"]}
-        for label in record.get("labels", []):
-            funder["title"][label["iso639"]] = label["label"]
+        funder["types"] = record.get("types")
+
+        status = record.get("status")
+        funder["status"] = status
 
         # The ROR is always listed in identifiers, expected by serialization
         funder["identifiers"] = [{"identifier": funder["id"], "scheme": "ror"}]
         valid_schemes = set(funder_schemes.keys())
         fund_ref = "fundref"
         valid_schemes.add(fund_ref)
-        for scheme, identifier in record.get("external_ids", {}).items():
-            scheme = scheme.lower()
+        for identifier in record.get("external_ids"):
+            scheme = identifier["type"]
             if scheme in valid_schemes:
                 value = identifier.get("preferred") or identifier.get("all")[0]
                 if scheme == fund_ref:
                     value = f"{funder_fundref_doi_prefix}/{value}"
                     scheme = "doi"
-
                 funder["identifiers"].append(
                     {
                         "identifier": value,
@@ -96,7 +131,7 @@ DATASTREAM_CONFIG = {
         {
             "type": "zip",
             "args": {
-                "regex": "(?<!_schema_v2)\\.json$",
+                "regex": "_schema_v2\\.json$",
             },
         },
         {"type": "json"},

invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json

@@ -9,7 +9,15 @@
     },
     "country": {
       "type": "string",
-      "description": "Represents a funder's origin country."
+      "description": "Represents a funder's origin country as a country code."
+    },
+    "country_name": {
+      "type": "string",
+      "description": "Represents a funder's origin country as a full name."
+    },
+    "location_name": {
+      "type": "string",
+      "description": "Represents a funder's location name (usually a city)."
     },
     "identifiers": {
       "description": "Alternate identifiers for the record.",
@@ -23,6 +31,30 @@
       "type": "string",
       "description": "Funders name."
     },
+    "acronym": {
+      "type": "string",
+      "description": "Acronym for funders name."
+    },
+    "status": {
+        "type": "string",
+        "description": "Status of the funder."
+    },
+    "aliases": {
+      "description": "Alternate names for the funder.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "types": {
+      "description": "Types of funders.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
     }

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json

@@ -57,6 +57,24 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json

@@ -57,6 +57,24 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json

@@ -57,6 +57,24 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/funders/schema.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -43,6 +44,8 @@ class FunderSchema(BaseVocabularySchema):
         required=True, validate=validate.Length(min=1, error=_("Name cannot be blank."))
     )
     country = SanitizedUnicode()
+    country_name = SanitizedUnicode()
+    location_name = SanitizedUnicode()
     identifiers = IdentifierSet(
         fields.Nested(
             partial(
@@ -57,6 +60,11 @@ class FunderSchema(BaseVocabularySchema):
         validate=validate.Length(min=1, error=_("PID cannot be blank."))
     )
 
+    acronym = SanitizedUnicode()
+    aliases = fields.List(SanitizedUnicode())
+    status = SanitizedUnicode()
+    types = fields.List(SanitizedUnicode())
+
     @validates_schema
     def validate_id(self, data, **kwargs):
         """Validates ID."""

invenio_vocabularies/datastreams/readers.py

@@ -15,7 +15,6 @@ import re
 import tarfile
 import zipfile
 from abc import ABC, abstractmethod
-from collections import defaultdict
 from json.decoder import JSONDecodeError
 
 import requests
@@ -80,7 +79,12 @@ class TarReader(BaseReader):
     def read(self, item=None, *args, **kwargs):
         """Opens a tar archive or uses the given file pointer."""
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, tarfile.TarFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a TarFile (e.g. if it is a BytesIO), try to create a TarFile from the item.
+                with tarfile.open(mode=self._mode, fileobj=item) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with tarfile.open(self._origin, self._mode) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)
@@ -136,7 +140,12 @@ class ZipReader(BaseReader):
         """Opens a Zip archive or uses the given file pointer."""
         # https://docs.python.org/3/library/zipfile.html
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, zipfile.ZipFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a ZipFile (e.g. if it is a BytesIO), try to create a ZipFile from the item.
+                with zipfile.ZipFile(item, **self._options) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with zipfile.ZipFile(self._origin, **self._options) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)

invenio_vocabularies/factories.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+"""Generate Vocabulary Config."""
+from copy import deepcopy
+
+import yaml
+from invenio_records_resources.proxies import current_service_registry
+
+from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
+from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
+from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
+
+
+class VocabularyConfig:
+    """Vocabulary Config Factory."""
+
+    config = None
+    vocabulary_name = None
+
+    def get_config(self, filepath=None, origin=None):
+        """Get the configuration for the vocabulary."""
+        config = deepcopy(self.config)
+        if filepath:
+            with open(filepath, encoding="utf-8") as f:
+                config = yaml.safe_load(f).get(self.vocabulary_name)
+        if origin:
+            config["readers"][0].setdefault("args", {})
+            config["readers"][0]["args"]["origin"] = origin
+        return config
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        return current_service_registry.get(self.vocabulary_name)
+
+
+class NamesVocabularyConfig(VocabularyConfig):
+    """Names Vocabulary Config."""
+
+    config = names_ds_config
+    vocabulary_name = "names"
+
+
+class FundersVocabularyConfig(VocabularyConfig):
+    """Funders Vocabulary Config."""
+
+    config = funders_ds_config
+    vocabulary_name = "funders"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Funders")
+
+
+class AwardsVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_ds_config
+    vocabulary_name = "awards"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Awards")
+
+
+def get_vocabulary_config(vocabulary):
+    """Factory function to get the appropriate Vocabulary Config."""
+    vocab_config = {
+        "names": NamesVocabularyConfig,
+        "funders": FundersVocabularyConfig,
+        "awards": AwardsVocabularyConfig,
+    }
+    return vocab_config.get(vocabulary, VocabularyConfig)()

invenio_vocabularies/services/tasks.py

@@ -11,6 +11,7 @@ from celery import shared_task
 from flask import current_app
 
 from ..datastreams.factories import DataStreamFactory
+from ..factories import get_vocabulary_config
 
 
 @shared_task(ignore_result=True)
@@ -26,3 +27,32 @@ def process_datastream(config):
         if result.errors:
             for err in result.errors:
                 current_app.logger.error(err)
+
+
+@shared_task()
+def import_funders():
+    """Import the funders vocabulary.
+
+    Only new records are imported.
+    Existing records are not updated.
+    """
+    vc = get_vocabulary_config("funders")
+    config = vc.get_config()
+
+    # When importing funders via a Celery task, make sure that we are automatically downloading the ROR file,
+    # instead of relying on a local file on the file system.
+    if config["readers"][0]["type"] == "ror-http":
+        readers_config_with_ror_http = config["readers"]
+    else:
+        readers_config_with_ror_http = [{"type": "ror-http"}] + config["readers"]
+
+    ds = DataStreamFactory.create(
+        readers_config=readers_config_with_ror_http,
+        transformers_config=config.get("transformers"),
+        writers_config=config["writers"],
+    )
+
+    for result in ds.process():
+        if result.errors:
+            for err in result.errors:
+                current_app.logger.error(err)

{invenio_vocabularies-3.4.1.dist-info → invenio_vocabularies-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 3.4.1
+Version: 4.0.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -11,7 +11,7 @@ Platform: any
 Classifier: Development Status :: 5 - Production/Stable
 Requires-Python: >=3.7
 Requires-Dist: invenio-i18n <3.0.0,>=2.0.0
-Requires-Dist: invenio-records-resources <6.0.0,>=5.0.0
+Requires-Dist: invenio-records-resources <7.0.0,>=6.0.0
 Requires-Dist: lxml >=4.5.0
 Requires-Dist: PyYAML >=5.4.1
 Provides-Extra: elasticsearch7
@@ -74,10 +74,15 @@ https://invenio-vocabularies.readthedocs.io/
 Changes
 =======
 
-Version 3.4.1 (released 2024-04-19)
-
-- funders: exclude ROR schema v2 json (#309)
+Version 4.0.0 (released 2024-06-04)
 
+- datastreams: implement factories for generating vocabulary configurations
+- datastreams: added ROR HTTP reader
+- funders: use ROR v2 dump instead of v1
+- datastreams: added celery task for funders using ROR HTTP reader
+- datastreams: add OpenAIRE Project HTTP Reader
+- datastreams: fix OpenAIRE graph dataset parsing
+- installation: upgrade invenio-records-resources
 
 Version 3.4.0 (released 2024-04-19)
 

{invenio_vocabularies-3.4.1.dist-info → invenio_vocabularies-4.0.0.dist-info}/RECORD

@@ -1,7 +1,8 @@
-invenio_vocabularies/__init__.py,sha256=XQVii6AE-S9xrNhp1WfDG0rzm6QbbjbisAMnQix5Qt4,377
-invenio_vocabularies/cli.py,sha256=Ymuy0l846eJXIA4UybunSqq7P9m2N0OdTtj6nEgd1-0,6355
+invenio_vocabularies/__init__.py,sha256=V5z9SuZ3KH-yIAm6O_EOrw27GQcnW3S76XWEknRVwBU,377
+invenio_vocabularies/cli.py,sha256=9Gfm3jjNDX3oQnYXupxSuqwXvWnJmsQ9GmyekJ1h5Kg,5239
 invenio_vocabularies/config.py,sha256=mLypkeVrPKZPtokvHSF-_Q7YcV4sCVONiyhGhu-34hI,3772
 invenio_vocabularies/ext.py,sha256=ukuvkhkLPBy2AITFLojLYTIUlP2qcbHNkt6ES8i1TwY,5310
+invenio_vocabularies/factories.py,sha256=iz1MzCwzZoj0cSxlYmr_RWSOL6gRXKHqJnFznshpcvQ,2421
 invenio_vocabularies/fixtures.py,sha256=nNWwH04HFASjfj1oy5kMdcQGKmVjzUuA5wSw-ER1QAg,1585
 invenio_vocabularies/proxies.py,sha256=H_bcJXPTwGbErx-pF89ChayOZcXJXJGZW7O0xkJkYQ4,693
 invenio_vocabularies/views.py,sha256=JCUA6yzF_160U1oAFUIgGt4MigsLQ9MiHOInHBQw98M,1381
@@ -23,7 +24,7 @@ invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/index.js,sha
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/index.js,sha256=iSx-bdQkKj6XA9NAam31bdcQmFygljQnjLcFjjK3lwU,245
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/index.js,sha256=7sSg482yJODQHU4jkP-hWJjpBOw7ubFr5nPZl5D_1gQ,262
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/AwardResults.js,sha256=AgqJg9GEcJvKZR4plZsH0j7cm9C3yjT9YCPI6uvmOyc,3499
-invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/CustomAwardForm.js,sha256=PDgCRtbOXkXOQCodbxoieHenhXrXhSTMQk4LcS-AjYg,4794
+invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/CustomAwardForm.js,sha256=DTEt6SV56bc7LC1KczUixHkdBUF3lGQs8rCc-YLii4Q,4804
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/FunderDropdown.js,sha256=pPMB9Hirc8z3efquecrO_CKlfXE1ws6OqDB7fB30l5k,2431
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/FundingField.js,sha256=aqQ92Z0WJ4rfXz0E8KIIN4-qjORdQYxXPa5eS5b_g8A,6912
 invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/forms/Funding/FundingField.test.js,sha256=z_4lWkf3h9Uin8OzSBSKY1wpUxtbnjE2kRUjW80jAhA,35
@@ -60,7 +61,7 @@ invenio_vocabularies/contrib/awards/__init__.py,sha256=KwCmwFalz-3pDs9iTa5TKUidB
 invenio_vocabularies/contrib/awards/api.py,sha256=OXukE7PLXs45BTtqVrhvGBNqLmQaI-CgXmHTCi36LZk,303
 invenio_vocabularies/contrib/awards/awards.py,sha256=N-BNE2ffUu-IlfKFBjMjfYqxL0vmpO8YMPe4q4e-rTo,2695
 invenio_vocabularies/contrib/awards/config.py,sha256=PlDHabkWDUzwa1Fvk_U2hG83kQYBqM1IyChg8Yg_VlY,1630
-invenio_vocabularies/contrib/awards/datastreams.py,sha256=wsr-H2qJ2TjFBGoHDx1eLkoDh1DcH9UOAKzXgMdtUpg,3849
+invenio_vocabularies/contrib/awards/datastreams.py,sha256=4BRifAmmEw0fUxo9JK5ZUXfSrSv1wllPvuGVOfick54,8244
 invenio_vocabularies/contrib/awards/models.py,sha256=mM-kSNf7kDH3oIbV8epxxbUi7muYqi4JreXxgWXlVzw,318
 invenio_vocabularies/contrib/awards/resources.py,sha256=_9YTqbhz8axFXGhG5y4WyjE27p9n-7e3c6HoBRditPA,411
 invenio_vocabularies/contrib/awards/schema.py,sha256=bg-09nvhEn-ipzaztK1iNbKc59saAM8v8hAW9zpKDAc,2786
@@ -75,26 +76,29 @@ invenio_vocabularies/contrib/awards/mappings/os-v2/__init__.py,sha256=9gRLFRtjhN
 invenio_vocabularies/contrib/awards/mappings/os-v2/awards/award-v1.0.0.json,sha256=KBAVvfuZDe_K8YISgbKCcpCZVVh-vkQIbt00OP1MPZA,1477
 invenio_vocabularies/contrib/awards/mappings/v7/__init__.py,sha256=fERdPp7K7ajaoUu_4_HVLfF-WD_qcl7QgST6yGb68s4,253
 invenio_vocabularies/contrib/awards/mappings/v7/awards/award-v1.0.0.json,sha256=KBAVvfuZDe_K8YISgbKCcpCZVVh-vkQIbt00OP1MPZA,1477
+invenio_vocabularies/contrib/common/__init__.py,sha256=DdbGYRthEpQtObhY_YK4vOT9Zf7FFagQ6qtUjJOYw-M,247
+invenio_vocabularies/contrib/common/ror/__init__.py,sha256=3u2-fre1SQ-4nz3Ay0nxj3ntmMZ8Ujh_4eV-fyxfmtc,239
+invenio_vocabularies/contrib/common/ror/datastreams.py,sha256=osd_-QarJkzsPQAAAK4LEs7BJLNtWUCGGuq1yztnnIU,2665
 invenio_vocabularies/contrib/funders/__init__.py,sha256=YxFXBDnT7NM8rFwxT_Ge3xXR2n17EM0alknQq7r_Bt8,478
 invenio_vocabularies/contrib/funders/api.py,sha256=QKGGeSnPHSoBfucvpaVruXT_txYidofZ080G3IxFkIo,306
-invenio_vocabularies/contrib/funders/config.py,sha256=-GncyorRv86fbM1WXi-w7X9QW7GqJz_uqs75Icx5Y3w,1704
-invenio_vocabularies/contrib/funders/datastreams.py,sha256=3jDLhRD_tAjvgwY_kEQMi4GLROKycgq3wURnZRJLmP0,3542
+invenio_vocabularies/contrib/funders/config.py,sha256=BbzCRIcPxSwDllhjnGmdec5A126Zs1I4ZyVUCGi3jRA,1756
+invenio_vocabularies/contrib/funders/datastreams.py,sha256=ezBtXebBmUmlPSLD8hA8BuR55kREtGShWeKTVXa3QUk,4995
 invenio_vocabularies/contrib/funders/facets.py,sha256=a068TVtt74Ncu0latb177LFK8EdnpbMOWecAKozA04M,1245
 invenio_vocabularies/contrib/funders/funders.py,sha256=nHqUoyRRW9oruN-ghmwzb9QIz_jyBgxNUpk41yvcso8,2292
 invenio_vocabularies/contrib/funders/models.py,sha256=RAU-_YVOUNVCn03_XGJ2czcVwXTaZPk5w7X_bMAgMOk,314
 invenio_vocabularies/contrib/funders/resources.py,sha256=He4gXd737ovdrHL-HB9dX7AGxp1BVJ9QteIO7JWUVSE,415
-invenio_vocabularies/contrib/funders/schema.py,sha256=REXr_Jh3No7lCLjOo-ZuCQqsrPjQwm-pIHV5GQ6-O64,2190
+invenio_vocabularies/contrib/funders/schema.py,sha256=tEOdMU2i0z_3PkXL1yD1qIJeoAy_8n22xxv8V7iltz0,2480
 invenio_vocabularies/contrib/funders/serializer.py,sha256=UwFsPnQiMUWx2lMQgh3Bv1uosubTUF5GFIAxwECNoAk,927
 invenio_vocabularies/contrib/funders/services.py,sha256=M-kQwtVOGygHe44vXr1L4E5Vvnpoco4KO0LRYwzhvMc,375
 invenio_vocabularies/contrib/funders/jsonschemas/__init__.py,sha256=O4XwUXCk-gx_K2LDVJOLkq07-ibsUkpT1vPaebgd0Gk,247
-invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json,sha256=V_E72iXF__W4SzxlHw8u3wQC9feZJWIalDRO95fTICw,783
+invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json,sha256=5Pyqf5ig16D2OnrFf1I3ws9PIShUPtZBrT8O-bfIMbw,1598
 invenio_vocabularies/contrib/funders/mappings/__init__.py,sha256=aSr-tZd9rsjet6leeS336gdSdZHXwZKdaPStNtVNQVk,244
 invenio_vocabularies/contrib/funders/mappings/os-v1/__init__.py,sha256=xXEX3tacmXp0I1KFtDw7ohIahozd2oIGp1UN40IhFic,251
-invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json,sha256=hxiPQnaZfb6V2v1GrrWiZAeBQRvlrAJ6-P-79wrJDho,1275
+invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json,sha256=HtYnmonywSGEov-1twath81ugIUFj-kV2oIsWB0JG5w,1595
 invenio_vocabularies/contrib/funders/mappings/os-v2/__init__.py,sha256=YvMRlKYTnEmyTzI9smZp_lO3w-zcK-8IpqT-jGUXEEY,251
-invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json,sha256=hxiPQnaZfb6V2v1GrrWiZAeBQRvlrAJ6-P-79wrJDho,1275
+invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json,sha256=HtYnmonywSGEov-1twath81ugIUFj-kV2oIsWB0JG5w,1595
 invenio_vocabularies/contrib/funders/mappings/v7/__init__.py,sha256=yFHmi3QYD65YKzLU5vMEtwAZn0gwkFYa6Db_tSbHjKE,254
-invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json,sha256=hxiPQnaZfb6V2v1GrrWiZAeBQRvlrAJ6-P-79wrJDho,1275
+invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json,sha256=HtYnmonywSGEov-1twath81ugIUFj-kV2oIsWB0JG5w,1595
 invenio_vocabularies/contrib/names/__init__.py,sha256=DBfsM7JMETZGaV5QmXEwE7zhCaAXvc2SZN6uXnW_V-c,451
 invenio_vocabularies/contrib/names/api.py,sha256=sEPn_jFX3gyoxgbdEUSIvOoPCUI8pocI6qCZO6mzCgQ,300
 invenio_vocabularies/contrib/names/config.py,sha256=hKDTEEBYGYOY6sMOArZjjkq2HJ6MJtRZp1geGLAFgRg,1735
@@ -135,7 +139,7 @@ invenio_vocabularies/datastreams/__init__.py,sha256=VPefh6k4Q3eYxKIW8I5zXUGucntp
 invenio_vocabularies/datastreams/datastreams.py,sha256=_jSXv2yAvSjt8btMoLJlqXOkqBzYb3Xe9m2GH50Nwag,3987
 invenio_vocabularies/datastreams/errors.py,sha256=IDUZ3gNtYGrhcOgApHCms1gNNJTyJzoMPmG5JtIeYNU,678
 invenio_vocabularies/datastreams/factories.py,sha256=js76i-cjcEzslgemsrwSzXQMy3SXtY3iFcsO-OlrMIM,2182
-invenio_vocabularies/datastreams/readers.py,sha256=iUA04MbuyEnTyQL8yImLCn6XLLTdGMssQnWf9yDZp8c,6768
+invenio_vocabularies/datastreams/readers.py,sha256=M4CRKyj7GH4iMernMqNRWgPlXjgVZTWD2X-Tp4spxEA,7409
 invenio_vocabularies/datastreams/transformers.py,sha256=wspny-kazYMRHjkkyFfRVNIYzJxbjAqokRCBQ_-gXcY,1357
 invenio_vocabularies/datastreams/writers.py,sha256=a85RYE6FAQqNF7obmlqHLW_ztJ1a6biS_i9a_yaKJw4,3615
 invenio_vocabularies/datastreams/xml.py,sha256=HFa-lfxj7kFrr2IjeN1jxSLDfcvpBwO9nZLZF2-BryE,997
@@ -167,7 +171,7 @@ invenio_vocabularies/services/permissions.py,sha256=one3NvNFYq-q15e6xxf85OkH1bWZ
 invenio_vocabularies/services/querystr.py,sha256=X3JHVF9B0O0iLWrnW3ok_bf_8jA-Cs_oAcYYkGOm3Uw,1829
 invenio_vocabularies/services/schema.py,sha256=ShnnH_ILHZGxE546J6Jsdwdeix6jLubSRomzf472DK8,4307
 invenio_vocabularies/services/service.py,sha256=W3wtKOttQjOr8Nkaus6m3KRuCMBqBsWUCAVv7Dj8bvM,7392
-invenio_vocabularies/services/tasks.py,sha256=zTAWdnI5celWBKrF986wQzCmkOTGOwTghtN7U5FMZ5Q,783
+invenio_vocabularies/services/tasks.py,sha256=rk-foPEhLL8kKoi8EPuUIJSjLX07NM3Fx8KjuY6Kqyk,1767
 invenio_vocabularies/services/custom_fields/__init__.py,sha256=QgvSsn-S1xLzbZ57pjjGTt5oI3HqzXHVjwGTtuPgzN8,421
 invenio_vocabularies/services/custom_fields/subject.py,sha256=k-qXbCjXPTk-Xo_W5sojFpQbkntv50odROiNyJHMzL8,2234
 invenio_vocabularies/services/custom_fields/vocabulary.py,sha256=eCvqrNloMMCCvqR49IQwzk2p4xapx5_bmQhd6ByJZFM,3019
@@ -267,10 +271,10 @@ invenio_vocabularies/translations/zh_CN/LC_MESSAGES/messages.mo,sha256=g1I5aNO8r
 invenio_vocabularies/translations/zh_CN/LC_MESSAGES/messages.po,sha256=vg8qC8ofpAdJ3mQz7mWM1ylKDpiNWXFs7rlMdSPkgKk,4629
 invenio_vocabularies/translations/zh_TW/LC_MESSAGES/messages.mo,sha256=cqSm8NtMAwrP9O6qbmtkDtRT1e9D93qpsJN5X9_PPVw,600
 invenio_vocabularies/translations/zh_TW/LC_MESSAGES/messages.po,sha256=9ACePz_EpB-LfcIJajZ2kp8Q04tcdrQLOtug162ZUss,4115
-invenio_vocabularies-3.4.1.dist-info/AUTHORS.rst,sha256=8d0p_WWE1r9DavvzMDi2D4YIGBHiMYcN3LYxqQOj8sY,291
-invenio_vocabularies-3.4.1.dist-info/LICENSE,sha256=UvI8pR8jGWqe0sTkb_hRG6eIrozzWwWzyCGEpuXX4KE,1062
-invenio_vocabularies-3.4.1.dist-info/METADATA,sha256=uBy7poGYcqlln0zyJcfHFu_wsECKRyt9E-jcmzOlWA4,6627
-invenio_vocabularies-3.4.1.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-invenio_vocabularies-3.4.1.dist-info/entry_points.txt,sha256=Cca3c5XGXkQItZ2-1oeN5xffwM8I1J3XjSvdq9-l_Bw,2497
-invenio_vocabularies-3.4.1.dist-info/top_level.txt,sha256=x1gRNbaODF_bCD0SBLM3nVOFPGi06cmGX5X94WKrFKk,21
-invenio_vocabularies-3.4.1.dist-info/RECORD,,
+invenio_vocabularies-4.0.0.dist-info/AUTHORS.rst,sha256=8d0p_WWE1r9DavvzMDi2D4YIGBHiMYcN3LYxqQOj8sY,291
+invenio_vocabularies-4.0.0.dist-info/LICENSE,sha256=UvI8pR8jGWqe0sTkb_hRG6eIrozzWwWzyCGEpuXX4KE,1062
+invenio_vocabularies-4.0.0.dist-info/METADATA,sha256=qTKnTHUD2J1jWkNiPAHYhDGDBLQaVvhT_j02lM1KtrE,6950
+invenio_vocabularies-4.0.0.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+invenio_vocabularies-4.0.0.dist-info/entry_points.txt,sha256=wQd9reAcv3btbEyMD9FHdkMbM7GvYDJiGugCuqc14sA,2589
+invenio_vocabularies-4.0.0.dist-info/top_level.txt,sha256=x1gRNbaODF_bCD0SBLM3nVOFPGi06cmGX5X94WKrFKk,21
+invenio_vocabularies-4.0.0.dist-info/RECORD,,

{invenio_vocabularies-3.4.1.dist-info → invenio_vocabularies-4.0.0.dist-info}/entry_points.txt

@@ -28,6 +28,9 @@ invenio_vocabularies_ext = invenio_vocabularies.views:blueprint
 [invenio_base.finalize_app]
 invenio_vocabularies = invenio_vocabularies.ext:finalize_app
 
+[invenio_celery.tasks]
+invenio_vocabularies_services = invenio_vocabularies.services.tasks
+
 [invenio_db.alembic]
 invenio_vocabularies = invenio_vocabularies:alembic