invenio-vocabularies 4.1.1__py2.py3-none-any.whl → 4.3.0__py2.py3-none-any.whl

invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "4.1.1"
+__version__ = "4.3.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

invenio_vocabularies/administration/views/vocabularies.py

@@ -8,6 +8,7 @@
 # details.
 
 """Vocabularies admin interface."""
+
 from invenio_administration.views.base import (
     AdminResourceEditView,
     AdminResourceListView,

invenio_vocabularies/cli.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -9,7 +9,6 @@
 
 """Commands to create and manage vocabularies."""
 
-
 import click
 from flask.cli import with_appcontext
 from invenio_access.permissions import system_identity
@@ -101,7 +100,10 @@ def update(vocabulary, filepath=None, origin=None):
     config = vc.get_config(filepath, origin)
 
     for w_conf in config["writers"]:
-        w_conf["args"]["update"] = True
+        if w_conf["type"] == "async":
+            w_conf["args"]["writer"]["args"]["update"] = True
+        else:
+            w_conf["args"]["update"] = True
 
     success, errored, filtered = _process_vocab(config)
 
@@ -140,18 +142,27 @@ def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=Non
     type=click.STRING,
     help="Identifier of the vocabulary item to delete.",
 )
-@click.option("--all", is_flag=True, default=False, help="Not supported yet.")
+@click.option("--all", is_flag=True, default=False)
 @with_appcontext
 def delete(vocabulary, identifier, all):
     """Delete all items or a specific one of the vocabulary."""
-    if not id and not all:
+    if not identifier and not all:
         click.secho("An identifier or the --all flag must be present.", fg="red")
         exit(1)
+
     vc = get_vocabulary_config(vocabulary)
     service = vc.get_service()
     if identifier:
         try:
-            if service.delete(identifier, system_identity):
+            if service.delete(system_identity, identifier):
                 click.secho(f"{identifier} deleted from {vocabulary}.", fg="green")
         except (PIDDeletedError, PIDDoesNotExistError):
             click.secho(f"PID {identifier} not found.")
+    elif all:
+        items = service.scan(system_identity)
+        for item in items.hits:
+            try:
+                if service.delete(system_identity, item["id"]):
+                    click.secho(f"{item['id']} deleted from {vocabulary}.", fg="green")
+            except (PIDDeletedError, PIDDoesNotExistError):
+                click.secho(f"PID {item['id']} not found.")

invenio_vocabularies/config.py

@@ -24,7 +24,7 @@ from .datastreams.readers import (
     ZipReader,
 )
 from .datastreams.transformers import XMLTransformer
-from .datastreams.writers import ServiceWriter, YamlWriter
+from .datastreams.writers import AsyncWriter, ServiceWriter, YamlWriter
 from .resources import VocabulariesResourceConfig
 from .services.config import VocabulariesServiceConfig
 
@@ -134,6 +134,7 @@ VOCABULARIES_DATASTREAM_TRANSFORMERS = {
 VOCABULARIES_DATASTREAM_WRITERS = {
     "service": ServiceWriter,
     "yaml": YamlWriter,
+    "async": AsyncWriter,
 }
 """Data Streams writers."""
 
@@ -154,3 +155,16 @@ VOCABULARIES_TYPES_SEARCH = {
     "sort": ["name", "count"],
 }
 """Vocabulary type search configuration."""
+
+VOCABULARIES_ORCID_ACCESS_KEY = "TODO"
+"""ORCID access key to access the s3 bucket."""
+VOCABULARIES_ORCID_SECRET_KEY = "TODO"
+"""ORCID secret key to access the s3 bucket."""
+VOCABULARIES_ORCID_SUMMARIES_BUCKET = "v3.0-summaries"
+"""ORCID summaries bucket name."""
+VOCABULARIES_ORCID_SYNC_MAX_WORKERS = 32
+"""ORCID max number of simultaneous workers/connections."""
+VOCABULARIES_ORCID_SYNC_SINCE = {
+    "days": 1,
+}
+"""ORCID time shift to sync. Parameters accepted are the ones passed to 'datetime.timedelta'."""

invenio_vocabularies/contrib/affiliations/api.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 Affiliation = record_type.record_cls

invenio_vocabularies/contrib/affiliations/datastreams.py

@@ -9,11 +9,11 @@
 
 """Affiliations datastreams, transformers, writers and readers."""
 
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
 from ...datastreams.writers import ServiceWriter
-from .config import affiliation_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class AffiliationsServiceWriter(ServiceWriter):
@@ -29,9 +29,35 @@ class AffiliationsServiceWriter(ServiceWriter):
         return entry["id"]
 
 
+class AffiliationsRORTransformer(RORTransformer):
+    """Affiliations ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_AFFILIATION_SCHEMES")
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Affiliations datastream readers."""
+
 VOCABULARIES_DATASTREAM_WRITERS = {
     "affiliations-service": AffiliationsServiceWriter,
 }
+"""Affiliations datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-affiliations": AffiliationsRORTransformer,
+}
+"""Affiliations datastream transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -46,17 +72,16 @@ DATASTREAM_CONFIG = {
     ],
     "transformers": [
         {
-            "type": "ror",
-            "args": {
-                "vocab_schemes": affiliation_schemes,
-            },
+            "type": "ror-affiliations",
         },
     ],
     "writers": [
         {
-            "type": "affiliations-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "affiliations-service",
+                }
             },
         }
     ],

invenio_vocabularies/contrib/affiliations/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 AffiliationsServiceConfig = record_type.service_config_cls

invenio_vocabularies/contrib/awards/awards.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary awards."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,

invenio_vocabularies/contrib/awards/datastreams.py

@@ -7,6 +7,7 @@
 # details.
 
 """Awards datastreams, transformers, writers and readers."""
+
 import io
 
 import requests

invenio_vocabularies/contrib/awards/services.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary awards."""
 
-
 from .awards import record_type
 
 AwardsServiceConfig = record_type.service_config_cls

invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -10,6 +10,7 @@
 """ROR-related Datastreams Readers/Writers/Transformers module."""
 
 import io
+from datetime import datetime
 
 import requests
 from idutils import normalize_ror
@@ -22,6 +23,11 @@ from invenio_vocabularies.datastreams.transformers import BaseTransformer
 class RORHTTPReader(BaseReader):
     """ROR HTTP Reader returning an in-memory binary stream of the latest ROR data dump ZIP file."""
 
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor."""
+        self._since = since
+        super().__init__(origin, mode, *args, **kwargs)
+
     def _iter(self, fp, *args, **kwargs):
         raise NotImplementedError(
             "RORHTTPReader downloads one file and therefore does not iterate through items"
@@ -34,15 +40,41 @@ class RORHTTPReader(BaseReader):
                 "RORHTTPReader does not support being chained after another reader"
             )
 
+        # Follow the DOI to get the link of the linkset
+        dataset_doi_link = "https://doi.org/10.5281/zenodo.6347574"
+        landing_page = requests.get(dataset_doi_link, allow_redirects=True)
+        landing_page.raise_for_status()
+
         # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the ROR data dump.
         # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
-        headers = {"Accept": "application/linkset+json"}
-        api_url = "https://zenodo.org/api/records/6347574"
-        api_resp = requests.get(api_url, headers=headers)
-        api_resp.raise_for_status()
+        if "linkset" not in landing_page.links:
+            raise ReaderError("Linkset not found in the ROR dataset record.")
+        linkset_response = requests.get(
+            landing_page.links["linkset"]["url"],
+            headers={"Accept": "application/linkset+json"},
+        )
+        linkset_response.raise_for_status()
+
+        if self._since:
+            for link in linkset_response.json()["linkset"]:
+                if "type" in link and link["type"] == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        link["anchor"], headers={"Accept": link["type"]}
+                    )
+                    json_ld_reponse.raise_for_status()
+
+                    # TODO Update to use dateCreated once the field is added to InvenioRDM. (https://github.com/inveniosoftware/invenio-rdm-records/issues/1777)
+                    last_dump_date = json_ld_reponse.json()["datePublished"]
+                    if datetime.fromisoformat(last_dump_date) < datetime.fromisoformat(
+                        self._since
+                    ):
+                        return
+                    break
+            else:
+                raise ReaderError("Couldn't find json-ld in publisher's linkset.")
 
         # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = api_resp.json()["linkset"][0]
+        landing_page_linkset = linkset_response.json()["linkset"][0]
 
         # Extract the URL of the only ZIP file linked to the record.
         landing_page_zip_items = [
@@ -164,3 +196,5 @@ class RORTransformer(BaseTransformer):
 VOCABULARIES_DATASTREAM_TRANSFORMERS = {
     "ror": RORTransformer,
 }
+
+VOCABULARIES_DATASTREAM_WRITERS = {}

invenio_vocabularies/contrib/funders/datastreams.py

@@ -9,12 +9,11 @@
 
 """Funders datastreams, transformers, writers and readers."""
 
-from idutils import normalize_ror
-from invenio_access.permissions import system_identity
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 
 from ...datastreams.writers import ServiceWriter
-from .config import funder_fundref_doi_prefix, funder_schemes
+from ..common.ror.datastreams import RORTransformer
 
 
 class FundersServiceWriter(ServiceWriter):
@@ -30,10 +29,40 @@ class FundersServiceWriter(ServiceWriter):
         return entry["id"]
 
 
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
+            )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
+
 VOCABULARIES_DATASTREAM_WRITERS = {
     "funders-service": FundersServiceWriter,
 }
-"""Funders Data Streams transformers."""
+"""Funders datastreams writers."""
+
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
+}
+"""Funders datastreams transformers."""
 
 
 DATASTREAM_CONFIG = {
@@ -48,18 +77,16 @@ DATASTREAM_CONFIG = {
     ],
     "transformers": [
         {
-            "type": "ror",
-            "args": {
-                "vocab_schemes": funder_schemes,
-                "funder_fundref_doi_prefix": funder_fundref_doi_prefix,
-            },
+            "type": "ror-funders",
         },
     ],
     "writers": [
         {
-            "type": "funders-service",
+            "type": "async",
             "args": {
-                "identity": system_identity,
+                "writer": {
+                    "type": "funders-service",
+                }
             },
         }
     ],

invenio_vocabularies/contrib/funders/funders.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Vocabulary funders."""
+
 from flask_resources import (
     BaseListSchema,
     JSONSerializer,

invenio_vocabularies/contrib/names/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,15 +8,123 @@
 
 """Names datastreams, transformers, writers and readers."""
 
+import csv
+import io
+import tarfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import timedelta
+
+import arrow
+import regex as re
+from flask import current_app
 from invenio_access.permissions import system_identity
 from invenio_records.dictutils import dict_lookup
 
+from invenio_vocabularies.contrib.names.s3client import S3OrcidClient
+
 from ...datastreams.errors import TransformerError
-from ...datastreams.readers import SimpleHTTPReader
+from ...datastreams.readers import BaseReader, SimpleHTTPReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
 
 
+class OrcidDataSyncReader(BaseReader):
+    """ORCiD Data Sync Reader."""
+
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor.
+
+        :param origin: Data source (e.g. filepath).
+                       Can be none in case of piped readers.
+        """
+        super().__init__(origin=origin, mode=mode, *args, **kwargs)
+        self.s3_client = S3OrcidClient()
+        self.since = since
+
+    def _fetch_orcid_data(self, orcid_to_sync, bucket):
+        """Fetches a single ORCiD record from S3."""
+        # The ORCiD file key is located in a folder which name corresponds to the last three digits of the ORCiD
+        suffix = orcid_to_sync[-3:]
+        key = f"{suffix}/{orcid_to_sync}.xml"
+        try:
+            return self.s3_client.read_file(f"s3://{bucket}/{key}")
+        except Exception as e:
+            # TODO: log
+            return None
+
+    def _process_lambda_file(self, fileobj):
+        """Process the ORCiD lambda file and returns a list of ORCiDs to sync.
+
+        The decoded fileobj looks like the following:
+        orcid, path, date_created, last_modified
+        0000-0001-5109-3700, http://orcid.org/0000-0001-5109-3700, 2014-08-02 15:00:00.000,2021-08-02 15:00:00.000
+
+        Yield ORCiDs to sync until the last sync date is reached.
+        """
+        date_format = "YYYY-MM-DD HH:mm:ss.SSSSSS"
+        date_format_no_millis = "YYYY-MM-DD HH:mm:ss"
+        time_shift = current_app.config["VOCABULARIES_ORCID_SYNC_SINCE"]
+        if self.since:
+            time_shift = self.since
+        last_sync = arrow.now() - timedelta(**time_shift)
+
+        file_content = fileobj.read().decode("utf-8")
+
+        csv_reader = csv.DictReader(file_content.splitlines())
+
+        for row in csv_reader:  # Skip the header line
+            orcid = row["orcid"]
+
+            # Lambda file is ordered by last modified date
+            last_modified_str = row["last_modified"]
+            try:
+                last_modified_date = arrow.get(last_modified_str, date_format)
+            except arrow.parser.ParserError:
+                last_modified_date = arrow.get(last_modified_str, date_format_no_millis)
+
+            if last_modified_date < last_sync:
+                break
+            yield orcid
+
+    def _iter(self, orcids):
+        """Iterates over the ORCiD records yielding each one."""
+        with ThreadPoolExecutor(
+            max_workers=current_app.config["VOCABULARIES_ORCID_SYNC_MAX_WORKERS"]
+        ) as executor:
+            futures = [
+                executor.submit(
+                    self._fetch_orcid_data,
+                    orcid,
+                    current_app.config["VOCABULARIES_ORCID_SUMMARIES_BUCKET"],
+                )
+                for orcid in orcids
+            ]
+            for future in as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    yield result
+
+    def read(self, item=None, *args, **kwargs):
+        """Streams the ORCiD lambda file, process it to get the ORCiDS to sync and yields it's data."""
+        # Read the file from S3
+        tar_content = self.s3_client.read_file(
+            "s3://orcid-lambda-file/last_modified.csv.tar"
+        )
+
+        orcids_to_sync = []
+        # Opens tar file and process it
+        with tarfile.open(fileobj=io.BytesIO(tar_content)) as tar:
+            # Iterate over each member (file or directory) in the tar file
+            for member in tar.getmembers():
+                # Extract the file
+                extracted_file = tar.extractfile(member)
+                if extracted_file:
+                    # Process the file and get the ORCiDs to sync
+                    orcids_to_sync.extend(self._process_lambda_file(extracted_file))
+
+        yield from self._iter(orcids_to_sync)
+
+
 class OrcidHTTPReader(SimpleHTTPReader):
     """ORCiD HTTP Reader."""
 
@@ -30,9 +138,26 @@ class OrcidHTTPReader(SimpleHTTPReader):
         super().__init__(origin, *args, **kwargs)
 
 
+DEFAULT_NAMES_EXCLUDE_REGEX = r"[\p{P}\p{S}\p{Nd}\p{No}\p{Emoji}--,.()\-']"
+"""Regex to filter out names with punctuations, symbols, decimal numbers and emojis."""
+
+
 class OrcidTransformer(BaseTransformer):
     """Transforms an ORCiD record into a names record."""
 
+    def __init__(
+        self, *args, names_exclude_regex=DEFAULT_NAMES_EXCLUDE_REGEX, **kwargs
+    ) -> None:
+        """Constructor."""
+        self._names_exclude_regex = names_exclude_regex
+        super().__init__()
+
+    def _is_valid_name(self, name):
+        """Check whether the name passes the regex."""
+        if not self._names_exclude_regex:
+            return True
+        return not bool(re.search(self._names_exclude_regex, name, re.UNICODE | re.V1))
+
     def apply(self, stream_entry, **kwargs):
         """Applies the transformation to the stream entry."""
         record = stream_entry.entry
@@ -42,6 +167,11 @@ class OrcidTransformer(BaseTransformer):
         name = person.get("name")
         if name is None:
             raise TransformerError(f"Name not found in ORCiD entry.")
+        if name.get("family-name") is None:
+            raise TransformerError(f"Family name not found in ORCiD entry.")
+
+        if not self._is_valid_name(name["given-names"] + name["family-name"]):
+            raise TransformerError(f"Invalid characters in name.")
 
         entry = {
             "id": orcid_id,
@@ -89,6 +219,7 @@ class NamesServiceWriter(ServiceWriter):
 
 VOCABULARIES_DATASTREAM_READERS = {
     "orcid-http": OrcidHTTPReader,
+    "orcid-data-sync": OrcidDataSyncReader,
 }
 
 
@@ -128,3 +259,30 @@ DATASTREAM_CONFIG = {
 
 An origin is required for the reader.
 """
+
+# TODO: Used on the jobs and should be set as a "PRESET" (naming to be defined)
+ORCID_PRESET_DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "orcid-data-sync",
+        },
+        {"type": "xml"},
+    ],
+    "transformers": [{"type": "orcid"}],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "names-service",
+                }
+            },
+        }
+    ],
+    "batch_size": 1000,
+    "write_many": True,
+}
+"""ORCiD Data Stream configuration.
+
+An origin is required for the reader.
+"""

invenio_vocabularies/contrib/names/s3client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""S3 client."""
+
+from flask import current_app
+
+try:
+    import s3fs
+except ImportError:
+    s3fs = None
+
+
+class S3Client:
+    """S3 client."""
+
+    def __init__(self, access_key, secret_key):
+        """Constructor."""
+        if s3fs is None:
+            raise Exception("s3fs is not installed.")
+
+        self.fs = s3fs.S3FileSystem(key=access_key, secret=secret_key)
+
+    def read_file(self, s3_path):
+        """Reads a file from S3."""
+        with self.fs.open(s3_path, "rb") as f:
+            return f.read()
+
+
+class S3OrcidClient(S3Client):
+    """S3 ORCiD client."""
+
+    def __init__(self):
+        """Constructor."""
+        super().__init__(
+            access_key=current_app.config["VOCABULARIES_ORCID_ACCESS_KEY"],
+            secret_key=current_app.config["VOCABULARIES_ORCID_SECRET_KEY"],
+        )