invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/funders/schema.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2024 California Institute of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -43,6 +44,8 @@ class FunderSchema(BaseVocabularySchema):
         required=True, validate=validate.Length(min=1, error=_("Name cannot be blank."))
     )
     country = SanitizedUnicode()
+    country_name = SanitizedUnicode()
+    location_name = SanitizedUnicode()
     identifiers = IdentifierSet(
         fields.Nested(
             partial(
@@ -57,6 +60,11 @@ class FunderSchema(BaseVocabularySchema):
         validate=validate.Length(min=1, error=_("PID cannot be blank."))
     )
 
+    acronym = SanitizedUnicode()
+    aliases = fields.List(SanitizedUnicode())
+    status = SanitizedUnicode()
+    types = fields.List(SanitizedUnicode())
+
     @validates_schema
     def validate_id(self, data, **kwargs):
         """Validates ID."""

invenio_vocabularies/contrib/funders/serializer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -29,4 +29,5 @@ class FunderL10NItemSchema(Schema):
     props = fields.Dict(dump_only=True)
     name = fields.String(dump_only=True)
     country = fields.String(dump_only=True)
+    country_name = fields.String(dump_only=True)
     identifiers = fields.List(fields.Nested(IdentifierSchema), dump_only=True)

invenio_vocabularies/contrib/names/config.py

@@ -28,12 +28,14 @@ class NamesSearchOptions(SearchOptions):
 
     suggest_parser_cls = SuggestQueryParser.factory(
         fields=[
-            "name^100",
-            "family_name^100",
             "given_name^100",
+            "name^70",
+            "family_name^50",
             "identifiers.identifier^20",
-            "affiliations.name^10",
+            "affiliations.name^20",
         ],
+        type="most_fields",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#multi-match-types
+        fuzziness="AUTO",
     )
 
     sort_default = "bestmatch"

invenio_vocabularies/contrib/names/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,18 +8,123 @@
 
 """Names datastreams, transformers, writers and readers."""
 
+import csv
+import io
+import tarfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import timedelta
+
+import arrow
+import regex as re
+from flask import current_app
 from invenio_access.permissions import system_identity
-from invenio_pidstore.errors import PIDDoesNotExistError
 from invenio_records.dictutils import dict_lookup
-from marshmallow import ValidationError
 
-from ...datastreams import StreamEntry
-from ...datastreams.errors import TransformerError, WriterError
-from ...datastreams.readers import SimpleHTTPReader
+from invenio_vocabularies.contrib.names.s3client import S3OrcidClient
+
+from ...datastreams.errors import TransformerError
+from ...datastreams.readers import BaseReader, SimpleHTTPReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
 
 
+class OrcidDataSyncReader(BaseReader):
+    """ORCiD Data Sync Reader."""
+
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor.
+
+        :param origin: Data source (e.g. filepath).
+                       Can be none in case of piped readers.
+        """
+        super().__init__(origin=origin, mode=mode, *args, **kwargs)
+        self.s3_client = S3OrcidClient()
+        self.since = since
+
+    def _fetch_orcid_data(self, orcid_to_sync, bucket):
+        """Fetches a single ORCiD record from S3."""
+        # The ORCiD file key is located in a folder which name corresponds to the last three digits of the ORCiD
+        suffix = orcid_to_sync[-3:]
+        key = f"{suffix}/{orcid_to_sync}.xml"
+        try:
+            return self.s3_client.read_file(f"s3://{bucket}/{key}")
+        except Exception as e:
+            # TODO: log
+            return None
+
+    def _process_lambda_file(self, fileobj):
+        """Process the ORCiD lambda file and returns a list of ORCiDs to sync.
+
+        The decoded fileobj looks like the following:
+        orcid, path, date_created, last_modified
+        0000-0001-5109-3700, http://orcid.org/0000-0001-5109-3700, 2014-08-02 15:00:00.000,2021-08-02 15:00:00.000
+
+        Yield ORCiDs to sync until the last sync date is reached.
+        """
+        date_format = "YYYY-MM-DD HH:mm:ss.SSSSSS"
+        date_format_no_millis = "YYYY-MM-DD HH:mm:ss"
+        time_shift = current_app.config["VOCABULARIES_ORCID_SYNC_SINCE"]
+        if self.since:
+            time_shift = self.since
+        last_sync = arrow.now() - timedelta(**time_shift)
+
+        file_content = fileobj.read().decode("utf-8")
+
+        csv_reader = csv.DictReader(file_content.splitlines())
+
+        for row in csv_reader:  # Skip the header line
+            orcid = row["orcid"]
+
+            # Lambda file is ordered by last modified date
+            last_modified_str = row["last_modified"]
+            try:
+                last_modified_date = arrow.get(last_modified_str, date_format)
+            except arrow.parser.ParserError:
+                last_modified_date = arrow.get(last_modified_str, date_format_no_millis)
+
+            if last_modified_date < last_sync:
+                break
+            yield orcid
+
+    def _iter(self, orcids):
+        """Iterates over the ORCiD records yielding each one."""
+        with ThreadPoolExecutor(
+            max_workers=current_app.config["VOCABULARIES_ORCID_SYNC_MAX_WORKERS"]
+        ) as executor:
+            futures = [
+                executor.submit(
+                    self._fetch_orcid_data,
+                    orcid,
+                    current_app.config["VOCABULARIES_ORCID_SUMMARIES_BUCKET"],
+                )
+                for orcid in orcids
+            ]
+            for future in as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    yield result
+
+    def read(self, item=None, *args, **kwargs):
+        """Streams the ORCiD lambda file, process it to get the ORCiDS to sync and yields it's data."""
+        # Read the file from S3
+        tar_content = self.s3_client.read_file(
+            "s3://orcid-lambda-file/last_modified.csv.tar"
+        )
+
+        orcids_to_sync = []
+        # Opens tar file and process it
+        with tarfile.open(fileobj=io.BytesIO(tar_content)) as tar:
+            # Iterate over each member (file or directory) in the tar file
+            for member in tar.getmembers():
+                # Extract the file
+                extracted_file = tar.extractfile(member)
+                if extracted_file:
+                    # Process the file and get the ORCiDs to sync
+                    orcids_to_sync.extend(self._process_lambda_file(extracted_file))
+
+        yield from self._iter(orcids_to_sync)
+
+
 class OrcidHTTPReader(SimpleHTTPReader):
     """ORCiD HTTP Reader."""
 
@@ -33,20 +138,43 @@ class OrcidHTTPReader(SimpleHTTPReader):
         super().__init__(origin, *args, **kwargs)
 
 
+DEFAULT_NAMES_EXCLUDE_REGEX = r"[\p{P}\p{S}\p{Nd}\p{No}\p{Emoji}--,.()\-']"
+"""Regex to filter out names with punctuations, symbols, decimal numbers and emojis."""
+
+
 class OrcidTransformer(BaseTransformer):
     """Transforms an ORCiD record into a names record."""
 
+    def __init__(
+        self, *args, names_exclude_regex=DEFAULT_NAMES_EXCLUDE_REGEX, **kwargs
+    ) -> None:
+        """Constructor."""
+        self._names_exclude_regex = names_exclude_regex
+        super().__init__()
+
+    def _is_valid_name(self, name):
+        """Check whether the name passes the regex."""
+        if not self._names_exclude_regex:
+            return True
+        return not bool(re.search(self._names_exclude_regex, name, re.UNICODE | re.V1))
+
     def apply(self, stream_entry, **kwargs):
         """Applies the transformation to the stream entry."""
         record = stream_entry.entry
         person = record["person"]
-        orcid_id = record["orcid-identifier"]["uri"]
+        orcid_id = record["orcid-identifier"]["path"]
 
         name = person.get("name")
         if name is None:
             raise TransformerError(f"Name not found in ORCiD entry.")
+        if name.get("family-name") is None:
+            raise TransformerError(f"Family name not found in ORCiD entry.")
+
+        if not self._is_valid_name(name["given-names"] + name["family-name"]):
+            raise TransformerError(f"Invalid characters in name.")
 
         entry = {
+            "id": orcid_id,
             "given_name": name.get("given-names"),
             "family_name": name.get("family-name"),
             "identifiers": [{"scheme": "orcid", "identifier": orcid_id}],
@@ -79,45 +207,19 @@ class OrcidTransformer(BaseTransformer):
 class NamesServiceWriter(ServiceWriter):
     """Names service writer."""
 
-    def __init__(self, *args, scheme_id="orcid", **kwargs):
+    def __init__(self, *args, **kwargs):
         """Constructor."""
         service_or_name = kwargs.pop("service_or_name", "names")
         super().__init__(service_or_name=service_or_name, *args, **kwargs)
-        self._scheme_id = scheme_id
 
     def _entry_id(self, entry):
         """Get the id from an entry."""
-        for identifier in entry.get("identifiers"):
-            if identifier.get("scheme") == self._scheme_id:
-                return identifier["identifier"]
-
-    def _resolve(self, id_):
-        """Resolve an entry given an id."""
-        return self._service.resolve(self._identity, id_=id_, id_type=self._scheme_id)
-
-    def write(self, stream_entry, *args, **kwargs):
-        """Writes the input entry using a given service."""
-        entry = stream_entry.entry
-        try:
-            vocab_id = self._entry_id(entry)
-            # it is resolved before creation to avoid duplicates since
-            # the pid is recidv2 not e.g. the orcid
-            current = self._resolve(vocab_id)
-            if not self._update:
-                raise WriterError([f"Vocabulary entry already exists: {entry}"])
-            updated = dict(current.to_dict(), **entry)
-            return StreamEntry(
-                self._service.update(self._identity, current.id, updated)
-            )
-        except PIDDoesNotExistError:
-            return StreamEntry(self._service.create(self._identity, entry))
-
-        except ValidationError as err:
-            raise WriterError([{"ValidationError": err.messages}])
+        return entry["id"]
 
 
 VOCABULARIES_DATASTREAM_READERS = {
     "orcid-http": OrcidHTTPReader,
+    "orcid-data-sync": OrcidDataSyncReader,
 }
 
 
@@ -138,10 +240,15 @@ DATASTREAM_CONFIG = {
         {
             "type": "tar",
             "args": {
-                "regex": ".xml$",
+                "regex": "\\.xml$",
+            },
+        },
+        {
+            "type": "xml",
+            "args": {
+                "root_element": "record",
             },
         },
-        {"type": "xml"},
     ],
     "transformers": [{"type": "orcid"}],
     "writers": [
@@ -157,3 +264,35 @@ DATASTREAM_CONFIG = {
 
 An origin is required for the reader.
 """
+
+# TODO: Used on the jobs and should be set as a "PRESET" (naming to be defined)
+ORCID_PRESET_DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "orcid-data-sync",
+        },
+        {
+            "type": "xml",
+            "args": {
+                "root_element": "record",
+            },
+        },
+    ],
+    "transformers": [{"type": "orcid"}],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "names-service",
+                }
+            },
+        }
+    ],
+    "batch_size": 1000,
+    "write_many": True,
+}
+"""ORCiD Data Stream configuration.
+
+An origin is required for the reader.
+"""

invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json

@@ -8,12 +8,8 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
-    "id": {
-      "description": "URI or classification code as identifier - globally unique among all names schemes.",
-      "$ref": "local://definitions-v1.0.0.json#/identifier"
-    },
-    "pid": {
-      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
     },
     "scheme": {
       "description": "Identifier of the name scheme.",

invenio_vocabularies/contrib/names/mappings/os-v1/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/names/mappings/os-v1/names/name-v2.0.0.json

@@ -0,0 +1,150 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercasepreserveoriginal": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifoldingpreserveoriginal": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "given_name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "family_name": {
+        "type": "text"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "affiliations": {
+        "type": "object",
+        "properties": {
+          "@v": {
+            "type": "keyword"
+          },
+          "id": {
+            "type": "keyword"
+          },
+          "name": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/names/mappings/os-v2/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },

invenio_vocabularies/contrib/names/mappings/os-v2/names/name-v2.0.0.json

@@ -0,0 +1,150 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercasepreserveoriginal": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifoldingpreserveoriginal": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "given_name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer"
+      },
+      "family_name": {
+        "type": "text"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "affiliations": {
+        "type": "object",
+        "properties": {
+          "@v": {
+            "type": "keyword"
+          },
+          "id": {
+            "type": "keyword"
+          },
+          "name": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/names/mappings/v7/names/name-v1.0.0.json

@@ -24,6 +24,9 @@
       "id": {
         "type": "keyword"
       },
+      "tags": {
+        "type": "keyword"
+      },
       "name_sort": {
         "type": "keyword"
       },