invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/contrib/common/openaire/datastreams.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""OpenAIRE-related Datastreams Readers/Writers/Transformers module."""
+
+import io
+
+import requests
+
+from invenio_vocabularies.datastreams.errors import ReaderError
+from invenio_vocabularies.datastreams.readers import BaseReader
+
+
+class OpenAIREHTTPReader(BaseReader):
+    """OpenAIRE HTTP Reader returning an in-memory binary stream of the latest OpenAIRE Graph Dataset tar file of a given type."""
+
+    def __init__(self, origin=None, mode="r", tar_href=None, *args, **kwargs):
+        """Constructor."""
+        self.tar_href = tar_href
+        super().__init__(origin, mode, *args, **kwargs)
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "OpenAIREHTTPReader downloads one file and therefore does not iterate through items"
+        )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest OpenAIRE Graph Dataset tar file of a given type from Zenodo and yields an in-memory binary stream of it."""
+        if item:
+            raise NotImplementedError(
+                "OpenAIREHTTPReader does not support being chained after another reader"
+            )
+
+        if self._origin == "full":
+            # OpenAIRE Graph Dataset
+            api_url = "https://zenodo.org/api/records/3516917"
+        elif self._origin == "diff":
+            # OpenAIRE Graph dataset: new collected projects
+            api_url = "https://zenodo.org/api/records/6419021"
+        else:
+            raise ReaderError("The --origin option should be either 'full' or 'diff'")
+
+        # Call the signposting `linkset+json` endpoint for the Concept DOI (i.e. latest version) of the OpenAIRE Graph Dataset.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        headers = {"Accept": "application/linkset+json"}
+        api_resp = requests.get(api_url, headers=headers)
+        api_resp.raise_for_status()
+
+        # Extract the Landing page Link Set Object located as the first (index 0) item.
+        landing_page_linkset = api_resp.json()["linkset"][0]
+
+        # Extract the URL of the only tar file matching `tar_href` linked to the record.
+        landing_page_matching_tar_items = [
+            item
+            for item in landing_page_linkset["item"]
+            if item["type"] == "application/x-tar"
+            and item["href"].endswith(self.tar_href)
+        ]
+        if len(landing_page_matching_tar_items) != 1:
+            raise ReaderError(
+                f"Expected 1 tar item matching {self.tar_href} but got {len(landing_page_matching_tar_items)}"
+            )
+        file_url = landing_page_matching_tar_items[0]["href"]
+
+        # Download the matching tar file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `tarfile.open`).
+        # Using directly `file_resp.raw` is not possible since `tarfile.open` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "openaire-http": OpenAIREHTTPReader,
+}
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {}
+
+VOCABULARIES_DATASTREAM_WRITERS = {}

invenio_vocabularies/contrib/common/ror/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related module."""

invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024-2025 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""ROR-related Datastreams Readers/Writers/Transformers module."""
+
+import io
+from datetime import datetime
+
+import requests
+from flask import current_app
+from idutils import normalize_ror
+
+from invenio_vocabularies.datastreams.errors import ReaderError, TransformerError
+from invenio_vocabularies.datastreams.readers import BaseReader
+from invenio_vocabularies.datastreams.transformers import BaseTransformer
+
+
+class RORHTTPReader(BaseReader):
+    """ROR HTTP Reader.
+
+    Returning an in-memory
+    binary stream of the latest ROR data dump ZIP file.
+    """
+
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Constructor."""
+        self._since = since
+        super().__init__(origin, mode, *args, **kwargs)
+
+    def _iter(self, fp, *args, **kwargs):
+        raise NotImplementedError(
+            "RORHTTPReader downloads one file "
+            "and therefore does not iterate through items"
+        )
+
+    def _get_last_dump_date(self, linksets):
+        """Get the last dump date."""
+        for linkset in linksets:
+            metadata_formats = linkset.get("describedby", [])
+            for format_link in metadata_formats:
+                if format_link.get("type") == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        format_link["href"],
+                        headers={"Accept": format_link["type"]},
+                    )
+                    json_ld_reponse.raise_for_status()
+                    json_ld_data = json_ld_reponse.json()
+
+                    last_dump_date = json_ld_data.get(
+                        "dateCreated"
+                    ) or json_ld_data.get("datePublished")
+                    last_dump_date = datetime.fromisoformat(
+                        last_dump_date.replace("Z", "+00:00")
+                    )
+                    return last_dump_date
+        else:
+            raise ReaderError(
+                "Couldn't find JSON-LD in publisher's linkset "
+                "to determine last dump date."
+            )
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads the latest ROR data dump.
+
+        Read from ZIP file from
+        Zenodo and yields an in-memory binary stream of it.
+        """
+        if item:
+            raise NotImplementedError(
+                "RORHTTPReader does not support being chained after another reader"
+            )
+
+        # Follow the DOI to get the link of the linkset
+        dataset_doi_link = "https://doi.org/10.5281/zenodo.6347574"
+        landing_page = requests.get(dataset_doi_link, allow_redirects=True)
+        landing_page.raise_for_status()
+
+        # Call the signposting `linkset+json` endpoint for
+        # the Concept DOI (i.e. latest version) of the ROR data dump.
+        # See: https://github.com/inveniosoftware/rfcs/blob/master/rfcs/rdm-0071-signposting.md#provide-an-applicationlinksetjson-endpoint
+        if "linkset" not in landing_page.links:
+            raise ReaderError("Linkset not found in the ROR dataset record.")
+        linkset_response = requests.get(
+            landing_page.links["linkset"]["url"],
+            headers={"Accept": "application/linkset+json"},
+        )
+        linkset_response.raise_for_status()
+        linksets = linkset_response.json()["linkset"]
+
+        if self._since and self._since != "None":
+            last_dump_date = self._get_last_dump_date(linksets)
+            since = datetime.fromisoformat(self._since)
+
+            if last_dump_date < since:
+                current_app.logger.info(
+                    f"Skipping ROR data dump (last dump: {last_dump_date}, since: {self._since})"
+                )
+                return
+
+        for linkset in linksets:
+            items = linkset.get("item", [])
+            zip_files = [item for item in items if item["type"] == "application/zip"]
+            if len(zip_files) == 1:
+                file_url = zip_files[0]["href"]
+                break
+            if len(zip_files) > 1:
+                raise ReaderError(f"Expected 1 ZIP item but got {len(zip_files)}")
+
+        current_app.logger.info(f"Reading ROR data dump (URL: {file_url})")
+
+        # Download the ZIP file and fully load the response bytes content in memory.
+        # The bytes content are then wrapped by a BytesIO to be
+        # file-like object (as required by `zipfile.ZipFile`).
+        # Using directly `file_resp.raw` is not possible since
+        # `zipfile.ZipFile` requires the file-like object to be seekable.
+        file_resp = requests.get(file_url)
+        file_resp.raise_for_status()
+        yield io.BytesIO(file_resp.content)
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "ror-http": RORHTTPReader,
+}
+
+
+class RORTransformer(BaseTransformer):
+    """Transforms a JSON ROR record into a funders record."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Initializes the transformer."""
+        self.vocab_schemes = vocab_schemes
+        self.funder_fundref_doi_prefix = funder_fundref_doi_prefix
+        super().__init__(*args, **kwargs)
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry."""
+        record = stream_entry.entry
+        ror = {}
+        ror["title"] = {}
+
+        ror["id"] = normalize_ror(record.get("id"))
+        if not ror["id"]:
+            raise TransformerError(_("Id not found in ROR entry."))
+
+        # Using set so aliases are unique
+        aliases = set()
+        acronym = None
+        for name in record.get("names"):
+            lang = name.get("lang", "en")
+            if lang == None:
+                lang = "en"
+            if "ror_display" in name["types"]:
+                ror["name"] = name["value"]
+            if "label" in name["types"]:
+                ror["title"][lang] = name["value"]
+            if "alias" in name["types"]:
+                aliases.add(name["value"])
+            if "acronym" in name["types"]:
+                # The first acronyn goes in acronym field to maintain
+                # compatability with existing data structure
+                if not acronym:
+                    acronym = name["value"]
+                else:
+                    aliases.add(name["value"])
+        if "en" not in ror["title"]:
+            ror["title"]["en"] = ror["name"]
+        if acronym:
+            ror["acronym"] = acronym
+        if aliases:
+            ror["aliases"] = list(aliases)
+
+        # ror_display is required and should be in every entry
+        if not ror["name"]:
+            raise TransformerError(
+                _("Name with type ror_display not found in ROR entry.")
+            )
+
+        # This only gets the first location, to maintain compatability
+        # with existing data structure
+        location = record.get("locations", [{}])[0].get("geonames_details", {})
+        ror["country"] = location.get("country_code")
+        ror["country_name"] = location.get("country_name")
+        ror["location_name"] = location.get("name")
+
+        ror["types"] = record.get("types")
+
+        status = record.get("status")
+        ror["status"] = status
+
+        # The ROR is always listed in identifiers, expected by serialization
+        ror["identifiers"] = [{"identifier": ror["id"], "scheme": "ror"}]
+        if self.vocab_schemes:
+            valid_schemes = set(self.vocab_schemes.keys())
+        else:
+            valid_schemes = set()
+        fund_ref = "fundref"
+        if self.funder_fundref_doi_prefix:
+            valid_schemes.add(fund_ref)
+        for identifier in record.get("external_ids"):
+            scheme = identifier["type"]
+            if scheme in valid_schemes:
+                value = identifier.get("preferred") or identifier.get("all")[0]
+                if scheme == fund_ref:
+                    if self.funder_fundref_doi_prefix:
+                        value = f"{self.funder_fundref_doi_prefix}/{value}"
+                        scheme = "doi"
+                ror["identifiers"].append(
+                    {
+                        "identifier": value,
+                        "scheme": scheme,
+                    }
+                )
+
+        stream_entry.entry = ror
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror": RORTransformer,
+}
+
+VOCABULARIES_DATASTREAM_WRITERS = {}

invenio_vocabularies/contrib/funders/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Funders vocabulary."""
+
+from .resources import FundersResource, FundersResourceConfig
+from .services import FundersService, FundersServiceConfig
+
+__all__ = [
+    "FundersResource",
+    "FundersResourceConfig",
+    "FundersService",
+    "FundersServiceConfig",
+]

invenio_vocabularies/contrib/funders/api.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary funders."""
+
+from .funders import record_type
+
+Funder = record_type.record_cls

invenio_vocabularies/contrib/funders/config.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary funders configuration."""
+
+from flask import current_app
+from invenio_i18n import get_locale
+from invenio_i18n import lazy_gettext as _
+from invenio_records_resources.services import SearchOptions
+from invenio_records_resources.services.records.components import DataComponent
+from invenio_records_resources.services.records.queryparser import (
+    CompositeSuggestQueryParser,
+)
+from werkzeug.local import LocalProxy
+
+from ...services.components import ModelPIDComponent
+
+funder_schemes = LocalProxy(lambda: current_app.config["VOCABULARIES_FUNDER_SCHEMES"])
+
+funder_fundref_doi_prefix = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_FUNDER_DOI_PREFIX"]
+)
+localized_title = LocalProxy(lambda: f"title.{get_locale()}^2")
+
+
+class FundersSearchOptions(SearchOptions):
+    """Search options."""
+
+    suggest_parser_cls = CompositeSuggestQueryParser.factory(
+        fields=[
+            # We boost the acronym fields, since they're smaller words and are more
+            # likely to be used in a query.
+            "acronym.keyword^50",
+            "acronym^10",
+            "name^10",
+            # Aliases can sometimes be shorter, so we boost them a bit.
+            "aliases^5",
+            localized_title,
+            # Allow to search identifiers directly (e.g. ROR)
+            "identifiers.identifier",
+            "country_name",
+        ],
+    )
+
+    sort_default = "bestmatch"
+
+    sort_default_no_query = "name"
+
+    sort_options = {
+        "bestmatch": dict(
+            title=_("Best match"),
+            fields=["_score"],  # ES defaults to desc on `_score` field
+        ),
+        "name": dict(
+            title=_("Name"),
+            fields=["name_sort"],
+        ),
+        "newest": dict(
+            title=_("Newest"),
+            fields=["-created"],
+        ),
+        "oldest": dict(
+            title=_("Oldest"),
+            fields=["created"],
+        ),
+    }
+
+
+service_components = [
+    # Order of components are important!
+    DataComponent,
+    ModelPIDComponent,
+]

invenio_vocabularies/contrib/funders/datastreams.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Funders datastreams, transformers, writers and readers."""
+
+from flask import current_app
+
+from ...datastreams.writers import ServiceWriter
+from ..common.ror.datastreams import RORTransformer
+
+
+class FundersServiceWriter(ServiceWriter):
+    """Funders service writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "funders")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+class FundersRORTransformer(RORTransformer):
+    """Funders ROR Transformer."""
+
+    def __init__(
+        self, *args, vocab_schemes=None, funder_fundref_doi_prefix=None, **kwargs
+    ):
+        """Constructor."""
+        if vocab_schemes is None:
+            vocab_schemes = current_app.config.get("VOCABULARIES_FUNDER_SCHEMES")
+        if funder_fundref_doi_prefix is None:
+            funder_fundref_doi_prefix = current_app.config.get(
+                "VOCABULARIES_FUNDER_DOI_PREFIX"
+            )
+        super().__init__(
+            *args,
+            vocab_schemes=vocab_schemes,
+            funder_fundref_doi_prefix=funder_fundref_doi_prefix,
+            **kwargs,
+        )
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""Funders datastreams writers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "funders-service": FundersServiceWriter,
+}
+"""Funders datastreams writers."""
+
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "ror-funders": FundersRORTransformer,
+}
+"""Funders datastreams transformers."""
+
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {"type": "ror-http"},
+        {
+            "type": "zip",
+            "args": {
+                "regex": "-ror-data\\.json$",
+            },
+        },
+        {"type": "json"},
+    ],
+    "transformers": [
+        {
+            "type": "ror-funders",
+        },
+    ],
+    "writers": [
+        {
+            "type": "async",
+            "args": {
+                "writer": {
+                    "type": "funders-service",
+                }
+            },
+        }
+    ],
+}
+"""Data Stream configuration.
+
+An origin is required for the reader.
+"""

invenio_vocabularies/contrib/funders/facets.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary awards facets and labels."""
+
+from ...services.facets import VocabularyLabels, lazy_get_label
+
+
+class FundersLabels(VocabularyLabels):
+    """Fetching of vocabulary labels for facets."""
+
+    def __init__(self, vocabulary, cache=True, cache_ttl=3600, service_id=None):
+        """Initialize the labels.
+
+        :param vocabulary: the name of the vocabulary type.
+        :param cache: use simple process in-memory cache when True.
+        :param cache_ttl: cache expiration in seconds.
+        :param service_id: the id of the registered service to be used
+            when fetching values for the vocabulary.
+        """
+        super().__init__(
+            vocabulary,
+            cache=cache,
+            cache_ttl=cache_ttl,
+            service_id="funders",
+        )
+        self.fields = ("id", "title", "country")  # not configurable
+
+    def _vocab_to_label(self, vocab):
+        """Returns the label string for a vocabulary entry."""
+        return f"{lazy_get_label(vocab['title'])} ({vocab['country']})"

invenio_vocabularies/contrib/funders/funders.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary funders."""
+
+from flask_resources import (
+    BaseListSchema,
+    JSONSerializer,
+    MarshmallowSerializer,
+    ResponseHandler,
+)
+from invenio_db import db
+from invenio_records.dumpers import SearchDumper
+from invenio_records.dumpers.indexedat import IndexedAtDumperExt
+from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.records.systemfields import ModelPIDField
+from invenio_records_resources.resources.records.headers import etag_headers
+
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
+from .config import FundersSearchOptions, service_components
+from .schema import FunderSchema
+from .serializer import FunderL10NItemSchema
+
+record_type = RecordTypeFactory(
+    "Funder",
+    # Data layer
+    pid_field_cls=ModelPIDField,
+    pid_field_kwargs={
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
+    },
+    record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
+        extensions=[
+            IndexedAtDumperExt(),
+        ],
+    ),
+    schema_version="1.0.0",
+    schema_path="local://funders/funder-v1.0.0.json",
+    index_name="funders-funder-v2.0.0",
+    # Service layer
+    service_id="funders",
+    service_schema=FunderSchema,
+    search_options=FundersSearchOptions,
+    service_components=service_components,
+    permission_policy_cls=PermissionPolicy,
+    # Resource layer
+    endpoint_route="/funders",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                MarshmallowSerializer(
+                    format_serializer_cls=JSONSerializer,
+                    object_schema_cls=FunderL10NItemSchema,
+                    list_schema_cls=BaseListSchema,
+                ),
+                headers=etag_headers,
+            ),
+        }
+    },
+)

invenio_vocabularies/contrib/funders/jsonschemas/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Funders JSONSchemas module."""