invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/funders/datastreams.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,14 +9,11 @@
 
 """Funders datastreams, transformers, writers and readers."""
 
-from idutils import normalize_ror
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
-from ...datastreams.errors import TransformerError
-from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
-from .config import funder_fundref_doi_prefix, funder_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class FundersServiceWriter(ServiceWriter):
@@ -31,64 +29,40 @@ class FundersServiceWriter(ServiceWriter):
         return entry["id"]
 
 
-class RORTransformer(BaseTransformer):
-    """Transforms a JSON ROR record into a funders record."""
-
-    def apply(self, stream_entry, **kwargs):
-        """Applies the transformation to the stream entry."""
-        record = stream_entry.entry
-        funder = {}
-
-        funder["id"] = normalize_ror(record.get("id"))
-        if not funder["id"]:
-            raise TransformerError(_(f"Id not found in ROR entry."))
-
-        funder["name"] = record.get("name")
-        if not funder["name"]:
-            raise TransformerError(_(f"Name not found in ROR entry."))
-
-        country_code = record.get("country", {}).get("country_code")
-        if country_code:
-            funder["country"] = country_code
-
-        funder["title"] = {"en": funder["name"]}
-        for label in record.get("labels", []):
-            funder["title"][label["iso639"]] = label["label"]
-
-        # The ROR is always listed in identifiers, expected by serialization
-        funder["identifiers"] = [{"identifier": funder["id"], "scheme": "ror"}]
-        valid_schemes = set(funder_schemes.keys())
-        fund_ref = "fundref"
-        valid_schemes.add(fund_ref)
-        for scheme, identifier in record.get("external_ids", {}).items():
-            scheme = scheme.lower()
-            if scheme in valid_schemes:
-                value = identifier.get("preferred") or identifier.get("all")[0]
-                if scheme == fund_ref:
-                    value = f"{funder_fundref_doi_prefix}/{value}"
-                    scheme = "doi"
-
-                funder["identifiers"].append(
-                    {
-                        "identifier": value,
-                        "scheme": scheme,
-                    }
-                )
-
-        stream_entry.entry = funder
-        return stream_entry
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
 
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
+            )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
 
-VOCABULARIES_DATASTREAM_TRANSFORMERS = {
-    "ror-funder": RORTransformer,
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "funders-service": FundersServiceWriter,
 }
-"""ROR Data Streams transformers."""
+"""Funders datastreams writers."""
 
 
-VOCABULARIES_DATASTREAM_WRITERS = {
-    "funders-service": FundersServiceWriter,
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
 }
-"""Funders Data Streams transformers."""
+"""Funders datastreams transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -96,19 +70,23 @@ DATASTREAM_CONFIG = {
         {
             "type": "zip",
             "args": {
-                "regex": ".json$",
+                "regex": "_schema_v2\\.json$",
             },
         },
         {"type": "json"},
     ],
     "transformers": [
-        {"type": "ror-funder"},
+        {
+            "type": "ror-funders",
+        },
     ],
     "writers": [
         {
-            "type": "funders-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "funders-service",
+                }
             },
         }
     ],

invenio_vocabularies/contrib/funders/facets.py

@@ -14,14 +14,22 @@ from ...services.facets import VocabularyLabels, lazy_get_label
 class FundersLabels(VocabularyLabels):
     """Fetching of vocabulary labels for facets."""
 
-    def __init__(self, vocabulary, cache=False, service_name=None):
-        """Initialize the labels."""
+    def __init__(self, vocabulary, cache=True, cache_ttl=3600, service_id=None):
+        """Initialize the labels.
+
+        :param vocabulary: the name of the vocabulary type.
+        :param cache: use simple process in-memory cache when True.
+        :param cache_ttl: cache expiration in seconds.
+        :param service_id: the id of the registered service to be used
+            when fetching values for the vocabulary.
+        """
         super().__init__(
             vocabulary,
-            cache,
-            service_name="funders",
+            cache=cache,
+            cache_ttl=cache_ttl,
+            service_id="funders",
         )
-        self.fields = ["id", "title", "country"]  # not configurable
+        self.fields = ("id", "title", "country")  # not configurable
 
     def _vocab_to_label(self, vocab):
         """Returns the label string for a vocabulary entry."""

invenio_vocabularies/contrib/funders/funders.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary funders."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,
@@ -35,7 +36,7 @@ record_type = RecordTypeFactory(
     model_cls_attrs={
         # cannot set to nullable=False because it would fail at
         # service level when create({}), see records-resources.
-        "pid": db.Column(db.String, unique=True),
+        "pid": db.Column(db.String(255), unique=True),
     },
     record_dumper=SearchDumper(
         model_fields={"pid": ("id", str)},
@@ -45,6 +46,7 @@ record_type = RecordTypeFactory(
     ),
     schema_version="1.0.0",
     schema_path="local://funders/funder-v1.0.0.json",
+    index_name="funders-funder-v2.0.0",
     # Service layer
     service_id="funders",
     service_schema=FunderSchema,

invenio_vocabularies/contrib/funders/jsonschemas/funders/funder-v1.0.0.json

@@ -7,9 +7,20 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "country": {
       "type": "string",
-      "description": "Represents a funder's origin country."
+      "description": "Represents a funder's origin country as a country code."
+    },
+    "country_name": {
+      "type": "string",
+      "description": "Represents a funder's origin country as a full name."
+    },
+    "location_name": {
+      "type": "string",
+      "description": "Represents a funder's location name (usually a city)."
     },
     "identifiers": {
       "description": "Alternate identifiers for the record.",
@@ -23,6 +34,30 @@
       "type": "string",
       "description": "Funders name."
     },
+    "acronym": {
+      "type": "string",
+      "description": "Acronym for funders name."
+    },
+    "status": {
+        "type": "string",
+        "description": "Status of the funder."
+    },
+    "aliases": {
+      "description": "Alternate names for the funder.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "types": {
+      "description": "Types of funders.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
     }

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v2.0.0.json

@@ -0,0 +1,156 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "types": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v2.0.0.json

@@ -0,0 +1,156 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "types": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/funders/mappings/v7/funders/funder-v1.0.0.json

@@ -57,12 +57,33 @@
       "country": {
         "type": "text"
       },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "acronym": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      },
       "id": {
         "type": "keyword"
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }