invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/affiliations/datastreams.py

@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Affiliations datastreams, transformers, writers and readers."""
+
+from flask import current_app
+
+from ...datastreams.errors import TransformerError, WriterError
+from ...datastreams.transformers import BaseTransformer
+from ...datastreams.writers import ServiceWriter
+from ..common.ror.datastreams import RORTransformer
+
+
+class AffiliationsServiceWriter(ServiceWriter):
+    """Affiliations service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "affiliations")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+class AffiliationsRORTransformer(RORTransformer):
+    """Affiliations ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_AFFILIATION_SCHEMES")
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+class OpenAIREOrganizationTransformer(BaseTransformer):
+    """OpenAIRE Organization Transformer."""
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+
+        if "id" not in record:
+            raise TransformerError([f"No id for: {record}"])
+
+        if not record["id"].startswith("openorgs____::"):
+            raise TransformerError([f"Not valid OpenAIRE OpenOrgs id for: {record}"])
+
+        if "pid" not in record:
+            raise TransformerError([f"No alternative identifiers for: {record}"])
+
+        organization = {}
+
+        for pid in record["pid"]:
+            if pid["scheme"] == "ROR":
+                organization["id"] = pid["value"].removeprefix("https://ror.org/")
+            elif pid["scheme"] == "PIC":
+                organization["identifiers"] = [
+                    {
+                        "scheme": "pic",
+                        "identifier": pid["value"],
+                    }
+                ]
+
+        stream_entry.entry = organization
+        return stream_entry
+
+
+class OpenAIREAffiliationsServiceWriter(ServiceWriter):
+    """OpenAIRE Affiliations service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "affiliations")
+        # Here we only update and we do not insert, since OpenAIRE data is used to augment existing affiliations
+        # (with PIC identifiers) and is not used to create new affiliations.
+        super().__init__(
+            service_or_name=service_or_name, insert=False, update=True, *args, **kwargs
+        )
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Writes the input entry using a given service."""
+        entry = stream_entry.entry
+
+        return super().write(stream_entry, *args, **kwargs)
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input entries using a given service."""
+        return super().write_many(stream_entries, *args, **kwargs)
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Affiliations datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "affiliations-service": AffiliationsServiceWriter,
+    "openaire-affiliations-service": OpenAIREAffiliationsServiceWriter,
+}
+"""Affiliations datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-affiliations": AffiliationsRORTransformer,
+    "openaire-organization": OpenAIREOrganizationTransformer,
+}
+"""Affiliations datastream transformers."""
+
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "zip",
+            "args": {
+                "regex": "_schema_v2\\.json$",
+            },
+        },
+        {"type": "json"},
+    ],
+    "transformers": [
+        {
+            "type": "ror-affiliations",
+        },
+    ],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "affiliations-service",
+                }
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""
+
+DATASTREAM_CONFIG_OPENAIRE = {
+    "readers": [
+        {"type": "openaire-http", "args": {"tar_href": "/organization.tar"}},
+        {
+            "type": "tar",
+            "args": {
+                "regex": "\\.json.gz$",
+                "mode": "r",
+            },
+        },
+        {"type": "gzip"},
+        {"type": "jsonl"},
+    ],
+    "transformers": [
+        {
+            "type": "openaire-organization",
+        },
+    ],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "openaire-affiliations-service",
+                }
+            },
+        }
+    ],
+}
+"""Alternative Data Stream configuration for OpenAIRE Affiliations."""

invenio_vocabularies/contrib/affiliations/facets.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary affiliations facets and labels."""
+
+from ...services.facets import VocabularyLabels, lazy_get_label
+
+
+class AffiliationsLabels(VocabularyLabels):
+    """Fetching of vocabulary labels for facets."""
+
+    def __init__(self, vocabulary, cache=True, cache_ttl=3600, service_id=None):
+        """Initialize the labels.
+
+        :param vocabulary: the name of the vocabulary type.
+        :param cache: use simple process in-memory cache when True.
+        :param cache_ttl: cache expiration in seconds.
+        :param service_id: the id of the registered service to be used
+            when fetching values for the vocabulary.
+        """
+        super().__init__(
+            vocabulary,
+            cache=cache,
+            cache_ttl=cache_ttl,
+            service_id="affiliations",
+        )
+        self.fields = ("id", "title")  # not configurable
+
+    def _vocab_to_label(self, vocab):
+        """Returns the label string for a vocabulary entry."""
+        return f"{lazy_get_label(vocab['title'])}"

invenio_vocabularies/contrib/affiliations/jsonschemas/affiliations/affiliation-v1.0.0.json

@@ -7,24 +7,55 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
+    "country": {
+      "type": "string",
+      "description": "Represents a affiliation's country as a country code."
+    },
+    "country_name": {
+      "type": "string",
+      "description": "Represents a affiliation's country as a full name."
+    },
+    "location_name": {
+      "type": "string",
+      "description": "Represents a affiliation's location name (usually a city)."
+    },
     "acronym": {
       "type": "string"
     },
-    "id": {
-      "$ref": "local://definitions-v1.0.0.json#/identifier"
-    },
     "identifiers": {
       "description": "Alternate identifiers for the record.",
       "type": "array",
-      "items": {"$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"},
+      "items": {
+        "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "status": {
+        "type": "string",
+        "description": "Status of the affiliation organization."
+    },
+    "aliases": {
+      "description": "Alternate names for the affiliation.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "types": {
+      "description": "Types of affiliation organization.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
       "uniqueItems": true
     },
     "name": {
       "type": "string"
     },
-    "pid": {
-      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
-    },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
     }

invenio_vocabularies/contrib/affiliations/mappings/os-v1/affiliations/affiliation-v1.0.0.json

@@ -84,7 +84,28 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },    
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/affiliations/mappings/os-v1/affiliations/affiliation-v2.0.0.json

@@ -0,0 +1,171 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercasepreserveoriginal": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifoldingpreserveoriginal": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/affiliations/mappings/os-v2/affiliations/affiliation-v1.0.0.json

@@ -84,7 +84,28 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/affiliations/mappings/os-v2/affiliations/affiliation-v2.0.0.json

@@ -0,0 +1,171 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercasepreserveoriginal": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifoldingpreserveoriginal": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      }
+    }
+  }
+}