invenio-vocabularies 5.1.0__tar.gz → 6.0.0__tar.gz

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/CHANGES.rst

@@ -8,6 +8,14 @@
 Changes
 =======
 
+Version v6.0.0 (released 2024-10-03)
+
+- datastreams: writers: add option to not insert
+- subjects: added euroscivoc datastream
+- affiliations: OpenAIRE transformer and writer adding PIC identifier
+- awards: added subjects and participating organizations from CORDIS datastreams
+- names: add permission check to names search
+
 Version v5.1.0 (released 2024-09-25)
 
 - funders: tune search boost for acronyms

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 5.1.0
+Version: 6.0.0
 Summary: Invenio module for managing vocabularies.
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,14 @@ Description: ..
         Changes
         =======
         
+        Version v6.0.0 (released 2024-10-03)
+        
+        - datastreams: writers: add option to not insert
+        - subjects: added euroscivoc datastream
+        - affiliations: OpenAIRE transformer and writer adding PIC identifier
+        - awards: added subjects and participating organizations from CORDIS datastreams
+        - names: add permission check to names search
+        
         Version v5.1.0 (released 2024-09-25)
         
         - funders: tune search boost for acronyms

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "5.1.0"
+__version__ = "6.0.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/invenio_vocabularies/cli.py

@@ -101,9 +101,14 @@ def update(vocabulary, filepath=None, origin=None):
 
     for w_conf in config["writers"]:
         if w_conf["type"] == "async":
-            w_conf["args"]["writer"]["args"]["update"] = True
+            w_conf_update = w_conf["args"]["writer"]
         else:
-            w_conf["args"]["update"] = True
+            w_conf_update = w_conf
+
+        if "args" in w_conf_update:
+            w_conf_update["args"]["update"] = True
+        else:
+            w_conf_update["args"] = {"update": True}
 
     success, errored, filtered = _process_vocab(config)
 

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/invenio_vocabularies/config.py

@@ -43,8 +43,17 @@ VOCABULARIES_IDENTIFIER_SCHEMES = {
 }
 """"Generic identifier schemes, usable by other vocabularies."""
 
+
+def is_pic(val):
+    """Test if argument is a Participant Identification Code (PIC)."""
+    if len(val) != 9:
+        return False
+    return val.isdigit()
+
+
 VOCABULARIES_AFFILIATION_SCHEMES = {
     **VOCABULARIES_IDENTIFIER_SCHEMES,
+    "pic": {"label": _("PIC"), "validator": is_pic},
 }
 """Affiliations allowed identifier schemes."""
 
@@ -107,6 +116,7 @@ VOCABULARIES_NAMES_SCHEMES = {
 
 VOCABULARIES_SUBJECTS_SCHEMES = {
     "gnd": {"label": _("GND"), "validator": idutils.is_gnd, "datacite": "GND"},
+    "url": {"label": _("URL"), "validator": idutils.is_url},
 }
 """Subjects allowed identifier schemes."""
 
@@ -162,6 +172,9 @@ VOCABULARIES_TYPES_SEARCH = {
 }
 """Vocabulary type search configuration."""
 
+SUBJECTS_EUROSCIVOC_FILE_URL = "https://op.europa.eu/o/opportal-service/euvoc-download-handler?cellarURI=http%3A%2F%2Fpublications.europa.eu%2Fresource%2Fdistribution%2Feuroscivoc%2F20231115-0%2Frdf%2Fskos_ap_eu%2FEuroSciVoc-skos-ap-eu.rdf&fileName=EuroSciVoc-skos-ap-eu.rdf"
+"""Subject EuroSciVoc file download link."""
+
 VOCABULARIES_ORCID_ACCESS_KEY = "TODO"
 """ORCID access key to access the s3 bucket."""
 VOCABULARIES_ORCID_SECRET_KEY = "TODO"

invenio-vocabularies-6.0.0/invenio_vocabularies/contrib/affiliations/datastreams.py

@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Affiliations datastreams, transformers, writers and readers."""
+
+from flask import current_app
+
+from ...datastreams.errors import TransformerError, WriterError
+from ...datastreams.transformers import BaseTransformer
+from ...datastreams.writers import ServiceWriter
+from ..common.ror.datastreams import RORTransformer
+
+
+class AffiliationsServiceWriter(ServiceWriter):
+    """Affiliations service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "affiliations")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+class AffiliationsRORTransformer(RORTransformer):
+    """Affiliations ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_AFFILIATION_SCHEMES")
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+class OpenAIREOrganizationTransformer(BaseTransformer):
+    """OpenAIRE Organization Transformer."""
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+
+        if "id" not in record:
+            raise TransformerError([f"No id for: {record}"])
+
+        if not record["id"].startswith("openorgs____::"):
+            raise TransformerError([f"Not valid OpenAIRE OpenOrgs id for: {record}"])
+
+        if "pid" not in record:
+            raise TransformerError([f"No alternative identifiers for: {record}"])
+
+        organization = {}
+
+        for pid in record["pid"]:
+            if pid["scheme"] == "ROR":
+                organization["id"] = pid["value"].removeprefix("https://ror.org/")
+            elif pid["scheme"] == "PIC":
+                organization["identifiers"] = [
+                    {
+                        "scheme": "pic",
+                        "identifier": pid["value"],
+                    }
+                ]
+
+        stream_entry.entry = organization
+        return stream_entry
+
+
+class OpenAIREAffiliationsServiceWriter(ServiceWriter):
+    """OpenAIRE Affiliations service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "affiliations")
+        # Here we only update and we do not insert, since OpenAIRE data is used to augment existing affiliations
+        # (with PIC identifiers) and is not used to create new affiliations.
+        super().__init__(
+            service_or_name=service_or_name, insert=False, update=True, *args, **kwargs
+        )
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Writes the input entry using a given service."""
+        entry = stream_entry.entry
+
+        return super().write(stream_entry, *args, **kwargs)
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input entries using a given service."""
+        return super().write_many(stream_entries, *args, **kwargs)
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Affiliations datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "affiliations-service": AffiliationsServiceWriter,
+    "openaire-affiliations-service": OpenAIREAffiliationsServiceWriter,
+}
+"""Affiliations datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-affiliations": AffiliationsRORTransformer,
+    "openaire-organization": OpenAIREOrganizationTransformer,
+}
+"""Affiliations datastream transformers."""
+
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "zip",
+            "args": {
+                "regex": "_schema_v2\\.json$",
+            },
+        },
+        {"type": "json"},
+    ],
+    "transformers": [
+        {
+            "type": "ror-affiliations",
+        },
+    ],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "affiliations-service",
+                }
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""
+
+DATASTREAM_CONFIG_OPENAIRE = {
+    "readers": [
+        {"type": "openaire-http", "args": {"tar_href": "/organization.tar"}},
+        {
+            "type": "tar",
+            "args": {
+                "regex": "\\.json.gz$",
+                "mode": "r",
+            },
+        },
+        {"type": "gzip"},
+        {"type": "jsonl"},
+    ],
+    "transformers": [
+        {
+            "type": "openaire-organization",
+        },
+    ],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "openaire-affiliations-service",
+                }
+            },
+        }
+    ],
+}
+"""Alternative Data Stream configuration for OpenAIRE Affiliations."""

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/invenio_vocabularies/contrib/awards/awards.py

@@ -18,24 +18,35 @@ from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records.dumpers.relations import RelationDumperExt
-from invenio_records.systemfields import RelationsField
+from invenio_records.systemfields import MultiRelationsField
 from invenio_records_resources.factories.factory import RecordTypeFactory
-from invenio_records_resources.records.systemfields import ModelPIDField, PIDRelation
+from invenio_records_resources.records.systemfields import (
+    ModelPIDField,
+    PIDListRelation,
+    PIDRelation,
+)
 from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...services.permissions import PermissionPolicy
 from ..funders.api import Funder
+from ..subjects.api import Subject
 from .config import AwardsSearchOptions, service_components
 from .schema import AwardSchema
 from .serializer import AwardL10NItemSchema
 
-award_relations = RelationsField(
+award_relations = MultiRelationsField(
     funders=PIDRelation(
         "funder",
         keys=["name"],
         pid_field=Funder.pid,
         cache_key="funder",
-    )
+    ),
+    subjects=PIDListRelation(
+        "subjects",
+        keys=["subject", "scheme", "identifiers", "props"],
+        pid_field=Subject.pid,
+        cache_key="subjects",
+    ),
 )
 
 record_type = RecordTypeFactory(

{invenio-vocabularies-5.1.0 → invenio-vocabularies-6.0.0}/invenio_vocabularies/contrib/awards/datastreams.py

@@ -11,70 +11,19 @@
 import io
 
 import requests
+from flask import current_app
 from invenio_access.permissions import system_identity
 from invenio_i18n import lazy_gettext as _
 
-from ...datastreams.errors import ReaderError, TransformerError
+from invenio_vocabularies.datastreams.errors import ReaderError
+
+from ...datastreams.errors import TransformerError
 from ...datastreams.readers import BaseReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
 from .config import awards_ec_ror_id, awards_openaire_funders_mapping
 
 
-class OpenAIREProjectHTTPReader(BaseReader):
-    """OpenAIRE Project HTTP Reader returning an in-memory binary stream of the latest OpenAIRE Graph Dataset project tar file."""
-
-    def _iter(self, fp, *args, **kwargs):
-        raise NotImplementedError(
-            "OpenAIREProjectHTTPReader downloads one file and therefore does not iterate through items"
-        )
-
-    def read(self, item=None, *args, **kwargs):
-        """Reads the latest OpenAIRE Graph Dataset project tar file from Zenodo and yields an in-memory binary stream of it."""
-        if item:
-            raise NotImplementedError(
-                "OpenAIREProjectHTTPReader does not support being chained after another reader"
-            )
-
-        if self._origin == "full":
-            # OpenAIRE Graph Dataset
-            api_url = "https://zenodo.org/api/records/3516917"
-        elif self._origin == "diff":
-            # OpenAIRE Graph dataset: new collected projects
-            api_url = "https://zenodo.org/api/records/6419021"
-        else:
-            raise ReaderError("The --origin option should be either 'full' or 'diff'")
-
-        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the OpenAIRE Graph Dataset.
-        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
-        headers = {"Accept": "application/linkset+json"}
-        api_resp = requests.get(api_url, headers=headers)
-        api_resp.raise_for_status()
-
-        # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = api_resp.json()["linkset"][0]
-
-        # Extract the URL of the only project tar file linked to the record.
-        landing_page_project_tar_items = [
-            item
-            for item in landing_page_linkset["item"]
-            if item["type"] == "application/x-tar"
-            and item["href"].endswith("/project.tar")
-        ]
-        if len(landing_page_project_tar_items) != 1:
-            raise ReaderError(
-                f"Expected 1 project tar item but got {len(landing_page_project_tar_items)}"
-            )
-        file_url = landing_page_project_tar_items[0]["href"]
-
-        # Download the project tar file and fully load the response bytes content in memory.
-        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `tarfile.open`).
-        # Using directly `file_resp.raw` is not possible since `tarfile.open` requires the file-like object to be seekable.
-        file_resp = requests.get(file_url)
-        file_resp.raise_for_status()
-        yield io.BytesIO(file_resp.content)
-
-
 class AwardsServiceWriter(ServiceWriter):
     """Funders service writer."""
 
@@ -125,10 +74,7 @@ class OpenAIREProjectTransformer(BaseTransformer):
 
         funding = next(iter(record.get("funding", [])), None)
         if funding:
-            funding_stream_id = funding.get("funding_stream", {}).get("id", "")
-            # Example funding stream ID: `EC::HE::HORIZON-AG-UN`. We need the `EC`
-            # string, i.e. the second "part" of the identifier.
-            program = next(iter(funding_stream_id.split("::")[1:2]), "")
+            program = funding.get("fundingStream", {}).get("id", "")
             if program:
                 award["program"] = program
 
@@ -172,20 +118,170 @@ class OpenAIREProjectTransformer(BaseTransformer):
         return stream_entry
 
 
+class CORDISProjectHTTPReader(BaseReader):
+    """CORDIS Project HTTP Reader returning an in-memory binary stream of the latest CORDIS Horizon Europe project zip file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "CORDISProjectHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest CORDIS Horizon Europe project zip file and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "CORDISProjectHTTPReader does not support being chained after another reader"
+            )
+
+        if self._origin == "HE":
+            file_url = "https://cordis.europa.eu/data/cordis-HORIZONprojects-xml.zip"
+        elif self._origin == "H2020":
+            file_url = "https://cordis.europa.eu/data/cordis-h2020projects-xml.zip"
+        elif self._origin == "FP7":
+            file_url = "https://cordis.europa.eu/data/cordis-fp7projects-xml.zip"
+        else:
+            raise ReaderError(
+                "The --origin option should be either 'HE' (for Horizon Europe) or 'H2020' (for Horizon 2020) or 'FP7'"
+            )
+
+        # Download the ZIP file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).
+        # Using directly `file_resp.raw` is not possible since `zipfile.ZipFile` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+class CORDISProjectTransformer(BaseTransformer):
+    """Transforms a CORDIS project record into an award record."""
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+        award = {}
+
+        # Here `id` is the project ID, which will be used to attach the update to the existing project.
+        award["id"] = (
+            f"{current_app.config['VOCABULARIES_AWARDS_EC_ROR_ID']}::{record['id']}"
+        )
+
+        categories = record.get("relations", {}).get("categories", {}).get("category")
+        if categories:
+            if isinstance(categories, dict):
+                categories = [categories]
+
+            award["subjects"] = [
+                {"id": f"euroscivoc:{vocab_id}"}
+                for category in categories
+                if category.get("@classification") == "euroSciVoc"
+                and (vocab_id := category["code"].split("/")[-1]).isdigit()
+            ]
+
+        organizations = (
+            record.get("relations", {}).get("associations", {}).get("organization")
+        )
+        if organizations:
+            # Projects with a single organization are not wrapped in a list,
+            # so we do this here to be able to iterate over it.
+            organizations = (
+                organizations if isinstance(organizations, list) else [organizations]
+            )
+            award["organizations"] = []
+            for organization in organizations:
+                # Some organizations in FP7 projects do not have a "legalname" key,
+                # for instance the 14th participant in "SAGE" https://cordis.europa.eu/project/id/999902.
+                # In this case, fully skip the organization entry.
+                if "legalname" not in organization:
+                    continue
+
+                organization_data = {
+                    "organization": organization["legalname"],
+                }
+
+                # Some organizations in FP7 projects do not have an "id" key (the PIC identifier),
+                # for instance "AIlGreenVehicles" in "MOTORBRAIN" https://cordis.europa.eu/project/id/270693.
+                # In this case, still store the name but skip the identifier part.
+                if "id" in organization:
+                    organization_data.update(
+                        {
+                            "scheme": "pic",
+                            "id": organization["id"],
+                        }
+                    )
+
+                award["organizations"].append(organization_data)
+
+        stream_entry.entry = award
+        return stream_entry
+
+
+class CORDISAwardsServiceWriter(ServiceWriter):
+    """CORDIS Awards service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "awards")
+        # Here we only update and we do not insert, since CORDIS data is used to augment existing awards
+        # (with subjects and organizations information) and is not used to create new awards.
+        super().__init__(
+            service_or_name=service_or_name, insert=False, update=True, *args, **kwargs
+        )
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
 VOCABULARIES_DATASTREAM_READERS = {
-    "openaire-project-http": OpenAIREProjectHTTPReader,
+    "cordis-project-http": CORDISProjectHTTPReader,
 }
 
 VOCABULARIES_DATASTREAM_TRANSFORMERS = {
     "openaire-award": OpenAIREProjectTransformer,
+    "cordis-award": CORDISProjectTransformer,
 }
 """ORCiD Data Streams transformers."""
 
 VOCABULARIES_DATASTREAM_WRITERS = {
     "awards-service": AwardsServiceWriter,
+    "cordis-awards-service": CORDISAwardsServiceWriter,
 }
 """ORCiD Data Streams transformers."""
 
+DATASTREAM_CONFIG_CORDIS = {
+    "readers": [
+        {"type": "cordis-project-http"},
+        {
+            "type": "zip",
+            "args": {
+                "regex": "\\.xml$",
+                "mode": "r",
+            },
+        },
+        {
+            "type": "xml",
+            "args": {
+                "root_element": "project",
+            },
+        },
+    ],
+    "transformers": [
+        {"type": "cordis-award"},
+    ],
+    "writers": [
+        {
+            "type": "cordis-awards-service",
+            "args": {
+                "identity": system_identity,
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""
+
 DATASTREAM_CONFIG = {
     "readers": [
         {

invenio-vocabularies-6.0.0/invenio_vocabularies/contrib/awards/jsonschemas/awards/award-v1.0.0.json

@@ -0,0 +1,102 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "local://awards/award-v1.0.0.json",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "$schema": {
+      "$ref": "local://definitions-v1.0.0.json#/$schema"
+    },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
+    "identifiers": {
+      "description": "Alternate identifiers for the award.",
+      "type": "array",
+      "items": {
+        "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "number": {
+      "type": "string"
+    },
+    "title": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "funder": {
+      "description": "Award's funder.",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "$ref": "local://definitions-v1.0.0.json#/identifier"
+        },
+        "name": {
+          "type": "string"
+        }
+      }
+    },
+    "acronym": {
+      "type": "string"
+    },
+    "program": {
+      "type": "string"
+    },
+    "subjects": {
+      "description": "Award's subjects.",
+      "type": "array",
+      "properties": {
+        "id": {
+          "$ref": "local://definitions-v1.0.0.json#/identifier"
+        },
+        "scheme": {
+          "description": "Identifier of the subject scheme.",
+          "$ref": "local://definitions-v1.0.0.json#/identifier"
+        },
+        "subject": {
+          "description": "Human readable label.",
+          "type": "string"
+        },
+        "props": {
+          "type": "object",
+          "patternProperties": {
+            "^.*$": {
+              "type": "string"
+            }
+          }
+        },
+        "identifiers": {
+          "description": "Alternate identifiers for the subject.",
+          "type": "array",
+          "items": {
+            "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "organizations": {
+      "description": "Award's organizations.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "scheme": {
+            "description": "Identifier of the organization scheme.",
+            "$ref": "local://definitions-v1.0.0.json#/identifier"
+          },
+          "id": {
+            "description": "Identifier of the organization for the given scheme.",
+            "$ref": "local://definitions-v1.0.0.json#/identifier"
+          },
+          "organization": {
+            "description": "Human readable label.",
+            "type": "string"
+          }
+        }
+      }
+    }
+  }
+}