invenio-vocabularies 2.3.1__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/affiliations/mappings/os-v2/affiliations/affiliation-v2.0.0.json

@@ -0,0 +1,171 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercasepreserveoriginal": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifoldingpreserveoriginal": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/affiliations/mappings/v7/affiliations/affiliation-v1.0.0.json

@@ -84,7 +84,28 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/affiliations/schema.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -11,7 +12,7 @@
 from functools import partial
 
 from invenio_i18n import lazy_gettext as _
-from marshmallow import fields
+from marshmallow import fields, validate
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
@@ -36,7 +37,20 @@ class AffiliationSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
             )
         )
     )
-    name = SanitizedUnicode(required=True)
+    name = SanitizedUnicode(
+        required=True, validate=validate.Length(min=1, error=_("Name cannot be blank."))
+    )
+    country = SanitizedUnicode()
+    country_name = SanitizedUnicode()
+    location_name = SanitizedUnicode()
+    id = SanitizedUnicode(
+        validate=validate.Length(min=1, error=_("PID cannot be blank."))
+    )
+
+    acronym = SanitizedUnicode()
+    aliases = fields.List(SanitizedUnicode())
+    status = SanitizedUnicode()
+    types = fields.List(SanitizedUnicode())
 
 
 class AffiliationRelationSchema(ContribVocabularyRelationSchema):

invenio_vocabularies/contrib/affiliations/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 AffiliationsServiceConfig = record_type.service_config_cls

invenio_vocabularies/contrib/awards/awards.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary awards."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,
@@ -17,24 +18,35 @@ from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records.dumpers.relations import RelationDumperExt
-from invenio_records.systemfields import RelationsField
+from invenio_records.systemfields import MultiRelationsField
 from invenio_records_resources.factories.factory import RecordTypeFactory
-from invenio_records_resources.records.systemfields import ModelPIDField, PIDRelation
+from invenio_records_resources.records.systemfields import (
+    ModelPIDField,
+    PIDListRelation,
+    PIDRelation,
+)
 from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...services.permissions import PermissionPolicy
 from ..funders.api import Funder
+from ..subjects.api import Subject
 from .config import AwardsSearchOptions, service_components
 from .schema import AwardSchema
 from .serializer import AwardL10NItemSchema
 
-award_relations = RelationsField(
+award_relations = MultiRelationsField(
     funders=PIDRelation(
         "funder",
         keys=["name"],
         pid_field=Funder.pid,
         cache_key="funder",
-    )
+    ),
+    subjects=PIDListRelation(
+        "subjects",
+        keys=["subject", "scheme", "identifiers", "props"],
+        pid_field=Subject.pid,
+        cache_key="subjects",
+    ),
 )
 
 record_type = RecordTypeFactory(

invenio_vocabularies/contrib/awards/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,10 +8,17 @@
 
 """Awards datastreams, transformers, writers and readers."""
 
+import io
+
+import requests
+from flask import current_app
 from invenio_access.permissions import system_identity
 from invenio_i18n import lazy_gettext as _
 
+from invenio_vocabularies.datastreams.errors import ReaderError
+
 from ...datastreams.errors import TransformerError
+from ...datastreams.readers import BaseReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
 from .config import awards_ec_ror_id, awards_openaire_funders_mapping
@@ -39,7 +46,20 @@ class OpenAIREProjectTransformer(BaseTransformer):
         award = {}
 
         code = record["code"]
-        openaire_funder_prefix = record["id"].split("::")[0].split("|")[1]
+
+        # The `id` should follow the format `sourcePrefix::md5(localId)` where `sourcePrefix` is 12 characters long.
+        # See: https://graph.openaire.eu/docs/data-model/pids-and-identifiers#identifiers-in-the-graph
+        #
+        # The format of `id` in the full OpenAIRE Graph Dataset (https://doi.org/10.5281/zenodo.3516917)
+        # follows this format (e.g. 'abc_________::0123456789abcdef0123456789abcdef').
+        # However, the format of `id` in the new collected projects dataset (https://doi.org/10.5281/zenodo.6419021)
+        # does not follow this format, and has a `40|` prefix (e.g. '40|abc_________::0123456789abcdef0123456789abcdef').
+        #
+        # The number '40' corresponds to the entity types 'Project'.
+        # See: https://ec.europa.eu/research/participants/documents/downloadPublic?documentIds=080166e5a3a1a213&appId=PPGMS
+        # See: https://graph.openaire.eu/docs/5.0.0/data-model/entities/project#id
+        openaire_funder_prefix = record["id"].split("::", 1)[0].split("|", 1)[-1]
+
         funder_id = awards_openaire_funders_mapping.get(openaire_funder_prefix)
         if funder_id is None:
             raise TransformerError(
@@ -54,10 +74,7 @@ class OpenAIREProjectTransformer(BaseTransformer):
 
         funding = next(iter(record.get("funding", [])), None)
         if funding:
-            funding_stream_id = funding.get("funding_stream", {}).get("id", "")
-            # Example funding stream ID: `EC::HE::HORIZON-AG-UN`. We need the `EC`
-            # string, i.e. the second "part" of the identifier.
-            program = next(iter(funding_stream_id.split("::")[1:2]), "")
+            program = funding.get("fundingStream", {}).get("id", "")
             if program:
                 award["program"] = program
 
@@ -78,7 +95,20 @@ class OpenAIREProjectTransformer(BaseTransformer):
             award["identifiers"] = identifiers
 
         award["number"] = code
+
+        # `title` is a mandatory attribute of the `Project` object in the OpenAIRE Graph Data Model.
+        # See: https://graph.openaire.eu/docs/data-model/entities/project#title
+        # However, 15'000+ awards for the FCT funder (and 1 record the NIH funder) are missing a title attribute.
+        if "title" not in record:
+            raise TransformerError(
+                _(
+                    "Missing title attribute for award {award_id}".format(
+                        award_id=award["id"]
+                    )
+                )
+            )
         award["title"] = {"en": record["title"]}
+
         award["funder"] = {"id": funder_id}
         acronym = record.get("acronym")
         if acronym:
@@ -88,22 +118,226 @@ class OpenAIREProjectTransformer(BaseTransformer):
         return stream_entry
 
 
+class CORDISProjectHTTPReader(BaseReader):
+    """CORDIS Project HTTP Reader returning an in-memory binary stream of the latest CORDIS Horizon Europe project zip file."""
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "CORDISProjectHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest CORDIS Horizon Europe project zip file and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "CORDISProjectHTTPReader does not support being chained after another reader"
+            )
+
+        if self._origin == "HE":
+            file_url = "https://cordis.europa.eu/data/cordis-HORIZONprojects-xml.zip"
+        elif self._origin == "H2020":
+            file_url = "https://cordis.europa.eu/data/cordis-h2020projects-xml.zip"
+        elif self._origin == "FP7":
+            file_url = "https://cordis.europa.eu/data/cordis-fp7projects-xml.zip"
+        else:
+            raise ReaderError(
+                "The --origin option should be either 'HE' (for Horizon Europe) or 'H2020' (for Horizon 2020) or 'FP7'"
+            )
+
+        # Download the ZIP file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).
+        # Using directly `file_resp.raw` is not possible since `zipfile.ZipFile` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+class CORDISProjectTransformer(BaseTransformer):
+    """Transforms a CORDIS project record into an award record."""
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+        award = {}
+
+        # Here `id` is the project ID, which will be used to attach the update to the existing project.
+        award["id"] = (
+            f"{current_app.config['VOCABULARIES_AWARDS_EC_ROR_ID']}::{record['id']}"
+        )
+
+        categories = record.get("relations", {}).get("categories", {}).get("category")
+        if categories:
+            if isinstance(categories, dict):
+                categories = [categories]
+
+            award["subjects"] = [
+                {"id": f"euroscivoc:{vocab_id}"}
+                for category in categories
+                if category.get("@classification") == "euroSciVoc"
+                and (vocab_id := category["code"].split("/")[-1]).isdigit()
+            ]
+
+        organizations = (
+            record.get("relations", {}).get("associations", {}).get("organization")
+        )
+        if organizations:
+            # Projects with a single organization are not wrapped in a list,
+            # so we do this here to be able to iterate over it.
+            organizations = (
+                organizations if isinstance(organizations, list) else [organizations]
+            )
+            award["organizations"] = []
+            for organization in organizations:
+                # Some organizations in FP7 projects do not have a "legalname" key,
+                # for instance the 14th participant in "SAGE" https://cordis.europa.eu/project/id/999902.
+                # In this case, fully skip the organization entry.
+                if "legalname" not in organization:
+                    continue
+
+                organization_data = {
+                    "organization": organization["legalname"],
+                }
+
+                # Some organizations in FP7 projects do not have an "id" key (the PIC identifier),
+                # for instance "AIlGreenVehicles" in "MOTORBRAIN" https://cordis.europa.eu/project/id/270693.
+                # In this case, still store the name but skip the identifier part.
+                if "id" in organization:
+                    organization_data.update(
+                        {
+                            "scheme": "pic",
+                            "id": organization["id"],
+                        }
+                    )
+
+                award["organizations"].append(organization_data)
+
+        programmes = (
+            record.get("relations", {}).get("associations", {}).get("programme", {})
+        )
+        if programmes:
+            # Projects with a single programme (this is the case of some projects in FP7) are not wrapped in a list,
+            # so we do this here to be able to iterate over it.
+            programmes = programmes if isinstance(programmes, list) else [programmes]
+
+            programmes_related_legal_basis = [
+                {
+                    "code": programme["code"],
+                    "uniqueprogrammepart": programme.get("@uniqueprogrammepart"),
+                }
+                for programme in programmes
+                if programme.get("@type") == "relatedLegalBasis"
+            ]
+
+            if len(programmes_related_legal_basis) == 0:
+                raise TransformerError(
+                    _(
+                        "No related legal basis programme found for project {project_id}".format(
+                            project_id=record["id"]
+                        )
+                    )
+                )
+            elif len(programmes_related_legal_basis) == 1:
+                # FP7 projects have only one related legal basis programme and do not have a 'uniqueprogrammepart' field.
+                unique_programme_related_legal_basis = programmes_related_legal_basis[0]
+            elif len(programmes_related_legal_basis) >= 1:
+                # The entry with the field 'uniqueprogrammepart' == 'true' is the high level programme code,
+                # while the other entry is a more specific sub-programme.
+                unique_programme_related_legal_basis = [
+                    programme_related_legal_basis
+                    # A few H2020 projects have more than one 'uniqueprogrammepart' == 'true',
+                    # for instance https://cordis.europa.eu/project/id/825673 (showing as "main programme" in the page)
+                    # which has one entry with the code 'H2020-EU.1.2.',
+                    # and one with the code 'H2020-EU.1.2.3.'.
+                    # We sort them from the shortest code to the longest code, and take the first item,
+                    # so that it conforms more with other projects which all have the shortest code as the main one.
+                    for programme_related_legal_basis in sorted(
+                        programmes_related_legal_basis, key=lambda d: len(d["code"])
+                    )
+                    if programme_related_legal_basis["uniqueprogrammepart"] == "true"
+                ][0]
+
+            # Store the code of the programme.
+            # For instance the code "HORIZON.1.2" which means "Marie Skłodowska-Curie Actions (MSCA)"
+            # See https://cordis.europa.eu/programme/id/HORIZON.1.2
+            award["program"] = unique_programme_related_legal_basis["code"]
+
+        stream_entry.entry = award
+        return stream_entry
+
+
+class CORDISAwardsServiceWriter(ServiceWriter):
+    """CORDIS Awards service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "awards")
+        # Here we only update and we do not insert, since CORDIS data is used to augment existing awards
+        # (with subjects, organizations, and program information) and is not used to create new awards.
+        super().__init__(
+            service_or_name=service_or_name, insert=False, update=True, *args, **kwargs
+        )
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "cordis-project-http": CORDISProjectHTTPReader,
+}
+
 VOCABULARIES_DATASTREAM_TRANSFORMERS = {
     "openaire-award": OpenAIREProjectTransformer,
+    "cordis-award": CORDISProjectTransformer,
 }
 """ORCiD Data Streams transformers."""
 
 VOCABULARIES_DATASTREAM_WRITERS = {
     "awards-service": AwardsServiceWriter,
+    "cordis-awards-service": CORDISAwardsServiceWriter,
 }
 """ORCiD Data Streams transformers."""
 
+DATASTREAM_CONFIG_CORDIS = {
+    "readers": [
+        {"type": "cordis-project-http"},
+        {
+            "type": "zip",
+            "args": {
+                "regex": "\\.xml$",
+                "mode": "r",
+            },
+        },
+        {
+            "type": "xml",
+            "args": {
+                "root_element": "project",
+            },
+        },
+    ],
+    "transformers": [
+        {"type": "cordis-award"},
+    ],
+    "writers": [
+        {
+            "type": "cordis-awards-service",
+            "args": {
+                "identity": system_identity,
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""
+
 DATASTREAM_CONFIG = {
     "readers": [
         {
             "type": "tar",
             "args": {
-                "regex": ".json.gz$",
+                "regex": "\\.json.gz$",
                 "mode": "r",
             },
         },

invenio_vocabularies/contrib/awards/jsonschemas/awards/award-v1.0.0.json

@@ -7,6 +7,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "identifiers": {
       "description": "Alternate identifiers for the award.",
       "type": "array",
@@ -39,6 +42,41 @@
     },
     "program": {
       "type": "string"
+    },
+    "subjects": {
+      "description": "Award's subjects.",
+      "type": "array",
+      "properties": {
+        "id": {
+          "$ref": "local://definitions-v1.0.0.json#/identifier"
+        },
+        "subject": {
+          "description": "Human readable label.",
+          "type": "string"
+        }
+      }
+    },
+    "organizations": {
+      "description": "Award's organizations.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "scheme": {
+            "description": "Identifier of the organization scheme.",
+            "$ref": "local://definitions-v1.0.0.json#/identifier"
+          },
+          "id": {
+            "description": "Identifier of the organization for the given scheme.",
+            "$ref": "local://definitions-v1.0.0.json#/identifier"
+          },
+          "organization": {
+            "description": "Human readable label.",
+            "type": "string"
+          }
+        }
+      }
     }
   }
 }