invenio-vocabularies 2.3.1__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/funders/datastreams.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,14 +9,11 @@
 
 """Funders datastreams, transformers, writers and readers."""
 
-from idutils import normalize_ror
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
-from ...datastreams.errors import TransformerError
-from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
-from .config import funder_fundref_doi_prefix, funder_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class FundersServiceWriter(ServiceWriter):
@@ -31,64 +29,40 @@ class FundersServiceWriter(ServiceWriter):
         return entry["id"]
 
 
-class RORTransformer(BaseTransformer):
-    """Transforms a JSON ROR record into a funders record."""
-
-    def apply(self, stream_entry, **kwargs):
-        """Applies the transformation to the stream entry."""
-        record = stream_entry.entry
-        funder = {}
-
-        funder["id"] = normalize_ror(record.get("id"))
-        if not funder["id"]:
-            raise TransformerError(_("Id not found in ROR entry."))
-
-        funder["name"] = record.get("name")
-        if not funder["name"]:
-            raise TransformerError(_("Name not found in ROR entry."))
-
-        country_code = record.get("country", {}).get("country_code")
-        if country_code:
-            funder["country"] = country_code
-
-        funder["title"] = {"en": funder["name"]}
-        for label in record.get("labels", []):
-            funder["title"][label["iso639"]] = label["label"]
-
-        # The ROR is always listed in identifiers, expected by serialization
-        funder["identifiers"] = [{"identifier": funder["id"], "scheme": "ror"}]
-        valid_schemes = set(funder_schemes.keys())
-        fund_ref = "fundref"
-        valid_schemes.add(fund_ref)
-        for scheme, identifier in record.get("external_ids", {}).items():
-            scheme = scheme.lower()
-            if scheme in valid_schemes:
-                value = identifier.get("preferred") or identifier.get("all")[0]
-                if scheme == fund_ref:
-                    value = f"{funder_fundref_doi_prefix}/{value}"
-                    scheme = "doi"
-
-                funder["identifiers"].append(
-                    {
-                        "identifier": value,
-                        "scheme": scheme,
-                    }
-                )
-
-        stream_entry.entry = funder
-        return stream_entry
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
 
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
+            )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
 
-VOCABULARIES_DATASTREAM_TRANSFORMERS = {
-    "ror-funder": RORTransformer,
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "funders-service": FundersServiceWriter,
 }
-"""ROR Data Streams transformers."""
+"""Funders datastreams writers."""
 
 
-VOCABULARIES_DATASTREAM_WRITERS = {
-    "funders-service": FundersServiceWriter,
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
 }
-"""Funders Data Streams transformers."""
+"""Funders datastreams transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -96,19 +70,23 @@ DATASTREAM_CONFIG = {
         {
             "type": "zip",
             "args": {
-                "regex": ".json$",
+                "regex": "_schema_v2\\.json$",
             },
         },
         {"type": "json"},
     ],
     "transformers": [
-        {"type": "ror-funder"},
+        {
+            "type": "ror-funders",
+        },
     ],
     "writers": [
         {
-            "type": "funders-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "funders-service",
+                }
             },
         }
     ],

invenio_vocabularies/contrib/funders/funders.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary funders."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,
@@ -45,6 +46,7 @@ record_type = RecordTypeFactory(
     ),
     schema_version="1.0.0",
     schema_path="local://funders/funder-v1.0.0.json",
+    index_name="funders-funder-v2.0.0",
     # Service layer
     service_id="funders",
     service_schema=FunderSchema,

invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json

@@ -7,9 +7,20 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "country": {
       "type": "string",
-      "description": "Represents a funder's origin country."
+      "description": "Represents a funder's origin country as a country code."
+    },
+    "country_name": {
+      "type": "string",
+      "description": "Represents a funder's origin country as a full name."
+    },
+    "location_name": {
+      "type": "string",
+      "description": "Represents a funder's location name (usually a city)."
     },
     "identifiers": {
       "description": "Alternate identifiers for the record.",
@@ -23,6 +34,30 @@
       "type": "string",
       "description": "Funders name."
     },
+    "acronym": {
+      "type": "string",
+      "description": "Acronym for funders name."
+    },
+    "status": {
+        "type": "string",
+        "description": "Status of the funder."
+    },
+    "aliases": {
+      "description": "Alternate names for the funder.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "types": {
+      "description": "Types of funders.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
     }

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v2.0.0.json

@@ -0,0 +1,156 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "types": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v2.0.0.json

@@ -0,0 +1,156 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "types": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/schema.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -43,6 +44,8 @@ class FunderSchema(BaseVocabularySchema):
         required=True, validate=validate.Length(min=1, error=_("Name cannot be blank."))
     )
     country = SanitizedUnicode()
+    country_name = SanitizedUnicode()
+    location_name = SanitizedUnicode()
     identifiers = IdentifierSet(
         fields.Nested(
             partial(
@@ -57,6 +60,11 @@ class FunderSchema(BaseVocabularySchema):
         validate=validate.Length(min=1, error=_("PID cannot be blank."))
     )
 
+    acronym = SanitizedUnicode()
+    aliases = fields.List(SanitizedUnicode())
+    status = SanitizedUnicode()
+    types = fields.List(SanitizedUnicode())
+
     @validates_schema
     def validate_id(self, data, **kwargs):
         """Validates ID."""

invenio_vocabularies/contrib/funders/serializer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -29,4 +29,5 @@ class FunderL10NItemSchema(Schema):
     props = fields.Dict(dump_only=True)
     name = fields.String(dump_only=True)
     country = fields.String(dump_only=True)
+    country_name = fields.String(dump_only=True)
     identifiers = fields.List(fields.Nested(IdentifierSchema), dump_only=True)