invenio-vocabularies 4.0.0__py2.py3-none-any.whl → 4.2.0__py2.py3-none-any.whl

invenio_vocabularies/contrib/awards/datastreams.py

@@ -7,6 +7,7 @@
 # details.
 
 """Awards datastreams, transformers, writers and readers."""
+
 import io
 
 import requests

invenio_vocabularies/contrib/awards/jsonschemas/awards/award-v1.0.0.json

@@ -7,6 +7,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "identifiers": {
       "description": "Alternate identifiers for the award.",
       "type": "array",

invenio_vocabularies/contrib/awards/mappings/os-v1/awards/award-v1.0.0.json

@@ -49,6 +49,9 @@
         "type": "object",
         "dynamic": "true"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "number": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/awards/mappings/os-v2/awards/award-v1.0.0.json

@@ -49,6 +49,9 @@
         "type": "object",
         "dynamic": "true"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "number": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/awards/mappings/v7/awards/award-v1.0.0.json

@@ -49,6 +49,9 @@
         "type": "object",
         "dynamic": "true"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "number": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/awards/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary awards."""
 
-
 from .awards import record_type
 
 AwardsServiceConfig = record_type.service_config_cls

invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,16 +10,24 @@
 """ROR-related Datastreams Readers/Writers/Transformers module."""
 
 import io
+from datetime import datetime
 
 import requests
+from idutils import normalize_ror
 
-from invenio_vocabularies.datastreams.errors import ReaderError
+from invenio_vocabularies.datastreams.errors import ReaderError, TransformerError
 from invenio_vocabularies.datastreams.readers import BaseReader
+from invenio_vocabularies.datastreams.transformers import BaseTransformer
 
 
 class RORHTTPReader(BaseReader):
     """ROR HTTP Reader returning an in-memory binary stream of the latest ROR data dump ZIP file."""
 
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor."""
+        self._since = since
+        super().__init__(origin, mode, *args, **kwargs)
+
     def _iter(self, fp, *args, **kwargs):
         raise NotImplementedError(
             "RORHTTPReader downloads one file and therefore does not iterate through items"
@@ -31,15 +40,41 @@ class RORHTTPReader(BaseReader):
                 "RORHTTPReader does not support being chained after another reader"
             )
 
+        # Follow the DOI to get the link of the linkset
+        dataset_doi_link = "https://doi.org/10.5281/zenodo.6347574"
+        landing_page = requests.get(dataset_doi_link, allow_redirects=True)
+        landing_page.raise_for_status()
+
         # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the ROR data dump.
         # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
-        headers = {"Accept": "application/linkset+json"}
-        api_url = "https://zenodo.org/api/records/6347574"
-        api_resp = requests.get(api_url, headers=headers)
-        api_resp.raise_for_status()
+        if "linkset" not in landing_page.links:
+            raise ReaderError("Linkset not found in the ROR dataset record.")
+        linkset_response = requests.get(
+            landing_page.links["linkset"]["url"],
+            headers={"Accept": "application/linkset+json"},
+        )
+        linkset_response.raise_for_status()
+
+        if self._since:
+            for link in linkset_response.json()["linkset"]:
+                if "type" in link and link["type"] == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        link["anchor"], headers={"Accept": link["type"]}
+                    )
+                    json_ld_reponse.raise_for_status()
+
+                    # TODO Update to use dateCreated once the field is added to InvenioRDM. (https://github.com/inveniosoftware/invenio-rdm-records/issues/1777)
+                    last_dump_date = json_ld_reponse.json()["datePublished"]
+                    if datetime.fromisoformat(last_dump_date) < datetime.fromisoformat(
+                        self._since
+                    ):
+                        return
+                    break
+            else:
+                raise ReaderError("Couldn't find json-ld in publisher's linkset.")
 
         # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = api_resp.json()["linkset"][0]
+        landing_page_linkset = linkset_response.json()["linkset"][0]
 
         # Extract the URL of the only ZIP file linked to the record.
         landing_page_zip_items = [
@@ -64,3 +99,102 @@ class RORHTTPReader(BaseReader):
 VOCABULARIES_DATASTREAM_READERS = {
     "ror-http": RORHTTPReader,
 }
+
+
+class RORTransformer(BaseTransformer):
+    """Transforms a JSON ROR record into a funders record."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Initializes the transformer."""
+        self.vocab_schemes = vocab_schemes
+        self.funder_fundref_doi_prefix = funder_fundref_doi_prefix
+        super().__init__(*args, **kwargs)
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+        ror = {}
+        ror["title"] = {}
+
+        ror["id"] = normalize_ror(record.get("id"))
+        if not ror["id"]:
+            raise TransformerError(_("Id not found in ROR entry."))
+
+        # Using set so aliases are unique
+        aliases = set()
+        acronym = None
+        for name in record.get("names"):
+            lang = name.get("lang", "en")
+            if lang == None:
+                lang = "en"
+            if "ror_display" in name["types"]:
+                ror["name"] = name["value"]
+            if "label" in name["types"]:
+                ror["title"][lang] = name["value"]
+            if "alias" in name["types"]:
+                aliases.add(name["value"])
+            if "acronym" in name["types"]:
+                # The first acronyn goes in acronym field to maintain
+                # compatability with existing data structure
+                if not acronym:
+                    acronym = name["value"]
+                else:
+                    aliases.add(name["value"])
+        if acronym:
+            ror["acronym"] = acronym
+        if aliases:
+            ror["aliases"] = list(aliases)
+
+        # ror_display is required and should be in every entry
+        if not ror["name"]:
+            raise TransformerError(
+                _("Name with type ror_display not found in ROR entry.")
+            )
+
+        # This only gets the first location, to maintain compatability
+        # with existing data structure
+        location = record.get("locations", [{}])[0].get("geonames_details", {})
+        ror["country"] = location.get("country_code")
+        ror["country_name"] = location.get("country_name")
+        ror["location_name"] = location.get("name")
+
+        ror["types"] = record.get("types")
+
+        status = record.get("status")
+        ror["status"] = status
+
+        # The ROR is always listed in identifiers, expected by serialization
+        ror["identifiers"] = [{"identifier": ror["id"], "scheme": "ror"}]
+        if self.vocab_schemes:
+            valid_schemes = set(self.vocab_schemes.keys())
+        else:
+            valid_schemes = set()
+        fund_ref = "fundref"
+        if self.funder_fundref_doi_prefix:
+            valid_schemes.add(fund_ref)
+        for identifier in record.get("external_ids"):
+            scheme = identifier["type"]
+            if scheme in valid_schemes:
+                value = identifier.get("preferred") or identifier.get("all")[0]
+                if scheme == fund_ref:
+                    if self.funder_fundref_doi_prefix:
+                        value = f"{self.funder_fundref_doi_prefix}/{value}"
+                        scheme = "doi"
+                ror["identifiers"].append(
+                    {
+                        "identifier": value,
+                        "scheme": scheme,
+                    }
+                )
+
+        stream_entry.entry = ror
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror": RORTransformer,
+}
+
+VOCABULARIES_DATASTREAM_WRITERS = {}

invenio_vocabularies/contrib/funders/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 # Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -9,14 +9,11 @@
 
 """Funders datastreams, transformers, writers and readers."""
 
-from idutils import normalize_ror
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
-from ...datastreams.errors import TransformerError
-from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
-from .config import funder_fundref_doi_prefix, funder_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class FundersServiceWriter(ServiceWriter):
@@ -32,98 +29,40 @@ class FundersServiceWriter(ServiceWriter):
         return entry["id"]
 
 
-class RORTransformer(BaseTransformer):
-    """Transforms a JSON ROR record into a funders record."""
-
-    def apply(self, stream_entry, **kwargs):
-        """Applies the transformation to the stream entry."""
-        record = stream_entry.entry
-        funder = {}
-        funder["title"] = {}
-
-        funder["id"] = normalize_ror(record.get("id"))
-        if not funder["id"]:
-            raise TransformerError(_("Id not found in ROR entry."))
-
-        aliases = []
-        acronym = None
-        for name in record.get("names"):
-            # Some name entries have a `lang` key with a `None` value.
-            # Therefore, providing a default value to `name.get("lang")` is not enough,
-            # and we need instead to check if the result of `get` is None.
-            lang = name.get("lang")
-            if lang is None:
-                lang = "en"
-
-            if "ror_display" in name["types"]:
-                funder["name"] = name["value"]
-            if "label" in name["types"]:
-                funder["title"][lang] = name["value"]
-            if "alias" in name["types"]:
-                aliases.append(name["value"])
-            if "acronym" in name["types"]:
-                # The first acronyn goes in acronym field to maintain
-                # compatability with existing data structure
-                if not acronym:
-                    acronym = name["value"]
-                else:
-                    aliases.append(name["value"])
-        if acronym:
-            funder["acronym"] = acronym
-        if aliases:
-            funder["aliases"] = aliases
-
-        # ror_display is required and should be in every entry
-        if not funder["name"]:
-            raise TransformerError(
-                _("Name with type ror_display not found in ROR entry.")
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
             )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs
+        )
 
-        # This only gets the first location, to maintain compatability
-        # with existing data structure
-        location = record.get("locations", [{}])[0].get("geonames_details", {})
-        funder["country"] = location.get("country_code")
-        funder["country_name"] = location.get("country_name")
-        funder["location_name"] = location.get("name")
-
-        funder["types"] = record.get("types")
-
-        status = record.get("status")
-        funder["status"] = status
-
-        # The ROR is always listed in identifiers, expected by serialization
-        funder["identifiers"] = [{"identifier": funder["id"], "scheme": "ror"}]
-        valid_schemes = set(funder_schemes.keys())
-        fund_ref = "fundref"
-        valid_schemes.add(fund_ref)
-        for identifier in record.get("external_ids"):
-            scheme = identifier["type"]
-            if scheme in valid_schemes:
-                value = identifier.get("preferred") or identifier.get("all")[0]
-                if scheme == fund_ref:
-                    value = f"{funder_fundref_doi_prefix}/{value}"
-                    scheme = "doi"
-                funder["identifiers"].append(
-                    {
-                        "identifier": value,
-                        "scheme": scheme,
-                    }
-                )
-
-        stream_entry.entry = funder
-        return stream_entry
 
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
 
-VOCABULARIES_DATASTREAM_TRANSFORMERS = {
-    "ror-funder": RORTransformer,
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "funders-service": FundersServiceWriter,
 }
-"""ROR Data Streams transformers."""
+"""Funders datastreams writers."""
 
 
-VOCABULARIES_DATASTREAM_WRITERS = {
-    "funders-service": FundersServiceWriter,
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
 }
-"""Funders Data Streams transformers."""
+"""Funders datastreams transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -137,13 +76,17 @@ DATASTREAM_CONFIG = {
         {"type": "json"},
     ],
     "transformers": [
-        {"type": "ror-funder"},
+        {
+            "type": "ror-funders",
+        },
     ],
     "writers": [
         {
-            "type": "funders-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "funders-service",
+                }
             },
         }
     ],

invenio_vocabularies/contrib/funders/funders.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary funders."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,

invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json

@@ -7,6 +7,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "country": {
       "type": "string",
       "description": "Represents a funder's origin country as a country code."

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json

@@ -81,6 +81,9 @@
       "title": {
         "type": "object",
         "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json

@@ -81,6 +81,9 @@
       "title": {
         "type": "object",
         "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json

@@ -81,6 +81,9 @@
       "title": {
         "type": "object",
         "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/serializer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -29,4 +29,5 @@ class FunderL10NItemSchema(Schema):
     props = fields.Dict(dump_only=True)
     name = fields.String(dump_only=True)
     country = fields.String(dump_only=True)
+    country_name = fields.String(dump_only=True)
     identifiers = fields.List(fields.Nested(IdentifierSchema), dump_only=True)

invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json

@@ -8,6 +8,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "scheme": {
       "description": "Identifier of the name scheme.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"

invenio_vocabularies/contrib/names/mappings/os-v1/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/names/mappings/os-v2/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/names/mappings/v7/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/subjects/jsonschemas/subjects/subject-v1.0.0.json

@@ -8,6 +8,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "id": {
       "description": "URI or classification code as identifier - globally unique among all subject schemes.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"

invenio_vocabularies/contrib/subjects/mappings/os-v1/subjects/subject-v1.0.0.json

@@ -55,6 +55,9 @@
             "type": "keyword"
           }
         }
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/os-v2/subjects/subject-v1.0.0.json

@@ -55,6 +55,9 @@
             "type": "keyword"
           }
         }
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/v7/subjects/subject-v1.0.0.json

@@ -55,6 +55,9 @@
             "type": "keyword"
           }
         }
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/datastreams/factories.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -36,7 +36,6 @@ class Factory:
         try:
             type_ = config["type"]
             args = config.get("args", {})
-
             return cls.options()[type_](**args)
         except KeyError:
             raise FactoryError(name=cls.FACTORY_NAME, key=type_)