invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/cli.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2021 Graz University of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Commands to create and manage vocabularies."""
+
+import click
+from flask.cli import with_appcontext
+from invenio_access.permissions import system_identity
+from invenio_pidstore.errors import PIDDeletedError, PIDDoesNotExistError
+
+from .datastreams import DataStreamFactory
+from .factories import get_vocabulary_config
+
+
+@click.group()
+def vocabularies():
+    """Vocabularies command."""
+
+
+def _process_vocab(config, num_samples=None):
+    """Import a vocabulary."""
+    ds = DataStreamFactory.create(
+        readers_config=config["readers"],
+        transformers_config=config.get("transformers"),
+        writers_config=config["writers"],
+        batch_size=config.get("batch_size", 1000),
+        write_many=config.get("write_many", False),
+    )
+
+    success, errored, filtered = 0, 0, 0
+    left = num_samples or -1
+    for result in ds.process():
+        left = left - 1
+        if result.filtered:
+            filtered += 1
+        if result.errors:
+            for err in result.errors:
+                click.secho(err, fg="red")
+            errored += 1
+        else:
+            success += 1
+        if left == 0:
+            click.secho(f"Number of samples reached {num_samples}", fg="green")
+            break
+    return success, errored, filtered
+
+
+def _output_process(vocabulary, op, success, errored, filtered):
+    """Outputs the result of an operation."""
+    total = success + errored
+
+    color = "green"
+    if errored:
+        color = "yellow" if success else "red"
+
+    click.secho(
+        f"Vocabulary {vocabulary} {op}. Total items {total}. \n"
+        f"{success} items succeeded\n"
+        f"{errored} contained errors\n"
+        f"{filtered} were filtered.",
+        fg=color,
+    )
+
+
+@vocabularies.command(name="import")
+@click.option("-v", "--vocabulary", type=click.STRING, required=True)
+@click.option("-f", "--filepath", type=click.STRING)
+@click.option("-o", "--origin", type=click.STRING)
+@click.option("-n", "--num-samples", type=click.INT)
+@with_appcontext
+def import_vocab(vocabulary, filepath=None, origin=None, num_samples=None):
+    """Import a vocabulary."""
+    if not filepath and not origin:
+        click.secho("One of --filepath or --origin must be present.", fg="red")
+        exit(1)
+
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+
+    success, errored, filtered = _process_vocab(config, num_samples)
+
+    _output_process(vocabulary, "imported", success, errored, filtered)
+
+
+@vocabularies.command()
+@click.option("-v", "--vocabulary", type=click.STRING, required=True)
+@click.option("-f", "--filepath", type=click.STRING)
+@click.option("-o", "--origin", type=click.STRING)
+@with_appcontext
+def update(vocabulary, filepath=None, origin=None):
+    """Import a vocabulary."""
+    if not filepath and not origin:
+        click.secho("One of --filepath or --origin must be present.", fg="red")
+        exit(1)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+
+    for w_conf in config["writers"]:
+        if w_conf["type"] == "async":
+            w_conf_update = w_conf["args"]["writer"]
+        else:
+            w_conf_update = w_conf
+
+        if "args" in w_conf_update:
+            w_conf_update["args"]["update"] = True
+        else:
+            w_conf_update["args"] = {"update": True}
+
+    success, errored, filtered = _process_vocab(config)
+
+    _output_process(vocabulary, "updated", success, errored, filtered)
+
+
+@vocabularies.command()
+@click.option("-v", "--vocabulary", type=click.STRING, required=True)
+@click.option("-f", "--filepath", type=click.STRING)
+@click.option("-o", "--origin", type=click.STRING)
+@click.option("-t", "--target", type=click.STRING)
+@click.option("-n", "--num-samples", type=click.INT)
+@with_appcontext
+def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=None):
+    """Convert a vocabulary to a new format."""
+    if not filepath and (not origin or not target):
+        click.secho(
+            "One of --filepath or --origin and --target must be present.", fg="red"
+        )
+        exit(1)
+
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+    if not filepath:
+        config["writers"] = [{"type": "yaml", "args": {"filepath": target}}]
+
+    success, errored, filtered = _process_vocab(config, num_samples)
+    _output_process(vocabulary, "converted", success, errored, filtered)
+
+
+@vocabularies.command()
+@click.option("-v", "--vocabulary", type=click.STRING, required=True)
+@click.option(
+    "-i",
+    "--identifier",
+    type=click.STRING,
+    help="Identifier of the vocabulary item to delete.",
+)
+@click.option("--all", is_flag=True, default=False)
+@with_appcontext
+def delete(vocabulary, identifier, all):
+    """Delete all items or a specific one of the vocabulary."""
+    if not identifier and not all:
+        click.secho("An identifier or the --all flag must be present.", fg="red")
+        exit(1)
+
+    vc = get_vocabulary_config(vocabulary)
+    service = vc.get_service()
+    if identifier:
+        try:
+            if service.delete(system_identity, identifier):
+                click.secho(f"{identifier} deleted from {vocabulary}.", fg="green")
+        except (PIDDeletedError, PIDDoesNotExistError):
+            click.secho(f"PID {identifier} not found.")
+    elif all:
+        items = service.scan(system_identity)
+        for item in items.hits:
+            try:
+                if service.delete(system_identity, item["id"]):
+                    click.secho(f"{item['id']} deleted from {vocabulary}.", fg="green")
+            except (PIDDeletedError, PIDDoesNotExistError):
+                click.secho(f"PID {item['id']} not found.")

invenio_vocabularies/config.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2025 CERN.
+# Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabularies configuration."""
+
+import re
+
+from idutils import is_doi, is_gnd, is_isni, is_orcid, is_ror, is_url
+from invenio_i18n import lazy_gettext as _
+
+from .datastreams.readers import (
+    CSVReader,
+    GzipReader,
+    JsonLinesReader,
+    JsonReader,
+    OAIPMHReader,
+    RDFReader,
+    SimpleHTTPReader,
+    SPARQLReader,
+    TarReader,
+    XMLReader,
+    YamlReader,
+    ZipReader,
+)
+from .datastreams.transformers import XMLTransformer
+from .datastreams.writers import AsyncWriter, ServiceWriter, YamlWriter
+from .resources import VocabulariesResourceConfig
+from .services.config import VocabulariesServiceConfig
+
+VOCABULARIES_RESOURCE_CONFIG = VocabulariesResourceConfig
+"""Configure the resource."""
+
+VOCABULARIES_SERVICE_CONFIG = VocabulariesServiceConfig
+"""Configure the service."""
+
+VOCABULARIES_IDENTIFIER_SCHEMES = {
+    "grid": {"label": _("GRID"), "validator": lambda x: True},
+    "gnd": {"label": _("GND"), "validator": is_gnd},
+    "isni": {"label": _("ISNI"), "validator": is_isni},
+    "ror": {"label": _("ROR"), "validator": is_ror},
+}
+""""Generic identifier schemes, usable by other vocabularies."""
+
+edmo_regexp = re.compile(r"^https://edmo\.seadatanet\.org/report/\d+$")
+
+
+def is_pic(val):
+    """Test if argument is a Participant Identification Code (PIC)."""
+    if len(val) != 9:
+        return False
+    return val.isdigit()
+
+
+def is_edmo(val):
+    """Test if argument is a European Directory of Marine Organisations (EDMO) identifier."""
+    return edmo_regexp.match(val)
+
+
+VOCABULARIES_AFFILIATION_SCHEMES = {
+    **VOCABULARIES_IDENTIFIER_SCHEMES,
+    "pic": {"label": _("PIC"), "validator": is_pic},
+    "edmo": {"label": _("EDMO"), "validator": is_edmo},
+}
+"""Affiliations allowed identifier schemes."""
+
+VOCABULARIES_FUNDER_SCHEMES = {
+    **VOCABULARIES_IDENTIFIER_SCHEMES,
+    "doi": {"label": _("DOI"), "validator": is_doi},
+}
+"""Funders allowed identifier schemes."""
+
+VOCABULARIES_FUNDER_DOI_PREFIX = "10.13039"
+"""DOI prefix for the identifier formed with the FundRef id."""
+
+VOCABULARIES_AWARD_SCHEMES = {
+    "url": {"label": _("URL"), "validator": is_url},
+    "doi": {"label": _("DOI"), "validator": is_doi},
+}
+"""Awards allowed identifier schemes."""
+
+VOCABULARIES_AWARDS_OPENAIRE_FUNDERS = {
+    "aka_________": "05k73zm37",
+    "anr_________": "00rbzpz17",
+    "arc_________": "05mmh0f86",
+    "asap________": "03zj4c476",
+    "cihr________": "01gavpb45",
+    "corda_______": "00k4n6c32",
+    "corda_____he": "00k4n6c32",
+    "corda__h2020": "00k4n6c32",
+    "euenvagency_": "02k4b9v70",
+    "fct_________": "00snfqn58",
+    "fwf_________": "013tf3c58",
+    "inca________": "03m8vkq32",
+    "irb_hr______": "03n51vw80",
+    "lcs_________": "02ar66p97",
+    "mestd_______": "01znas443",
+    "nhmrc_______": "011kf5r70",
+    "nih_________": "01cwqze88",
+    "nserc_______": "01h531d29",
+    "nsf_________": "021nxhr62",
+    "nwo_________": "04jsz6e67",
+    "rcuk________": "00dq2kk65",  # deprecated funder org
+    "sfi_________": "0271asj38",
+    "snsf________": "00yjd3n13",
+    "sshrc_______": "006cvnv84",
+    "tubitakf____": "04w9kkr77",
+    "twcf________": "00x0z1472",
+    "ukri________": "001aqnf71",
+    "wt__________": "029chgv08",
+}
+"""Mapping of OpenAIRE and ROR funder codes."""
+
+VOCABULARIES_AWARDS_EC_ROR_ID = "00k4n6c32"
+"""ROR ID for EC funder."""
+
+VOCABULARIES_NAMES_SCHEMES = {
+    "orcid": {"label": _("ORCID"), "validator": is_orcid, "datacite": "ORCID"},
+    "isni": {"label": _("ISNI"), "validator": is_isni, "datacite": "ISNI"},
+    "gnd": {"label": _("GND"), "validator": is_gnd, "datacite": "GND"},
+}
+"""Names allowed identifier schemes."""
+
+VOCABULARIES_SUBJECTS_SCHEMES = {
+    "gnd": {"label": _("GND"), "validator": is_gnd, "datacite": "GND"},
+    "url": {"label": _("URL"), "validator": is_url},
+}
+"""Subjects allowed identifier schemes."""
+
+VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
+    "names",
+    "affiliations",
+    "awards",
+    "funders",
+    "subjects",
+]
+"""List of custom vocabulary types."""
+
+VOCABULARIES_DATASTREAM_READERS = {
+    "csv": CSVReader,
+    "json": JsonReader,
+    "jsonl": JsonLinesReader,
+    "gzip": GzipReader,
+    "tar": TarReader,
+    "http": SimpleHTTPReader,
+    "rdf": RDFReader,
+    "sparql": SPARQLReader,
+    "yaml": YamlReader,
+    "zip": ZipReader,
+    "xml": XMLReader,
+    "oai-pmh": OAIPMHReader,
+}
+"""Data Streams readers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "xml": XMLTransformer,
+}
+"""Data Streams transformers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "service": ServiceWriter,
+    "yaml": YamlWriter,
+    "async": AsyncWriter,
+}
+"""Data Streams writers."""
+
+VOCABULARIES_TYPES_SORT_OPTIONS = {
+    "name": dict(
+        title=_("Name"),
+        fields=["id"],
+    ),
+    "count": dict(
+        title=_("Number of entries"),
+        fields=["count"],
+    ),
+}
+"""Definitions of available Vocabulary types sort options. """
+
+VOCABULARIES_TYPES_SEARCH = {
+    "facets": [],
+    "sort": ["name", "count"],
+}
+"""Vocabulary type search configuration."""
+
+VOCABULARIES_SUBJECTS_EUROSCIVOC_FILE_URL = "https://publications.europa.eu/resource/distribution/euroscivoc/rdf/skos_ap_eu/EuroSciVoc-skos-ap-eu.rdf"
+"""Subject EuroSciVoc file download link."""
+
+VOCABULARIES_SUBJECTS_GEMET_FILE_URL = (
+    "https://www.eionet.europa.eu/gemet/latest/gemet.rdf.gz"
+)
+"""Subject GEMET file download link."""
+
+VOCABULARIES_SUBJECTS_NVS_FILE_URL = "http://vocab.nerc.ac.uk/collection/P02/current/?_profile=nvs&_mediatype=application/rdf+xml"
+"""Subject NVS-P02 file download link."""
+
+VOCABULARIES_AFFILIATIONS_EDMO_COUNTRY_MAPPING = {
+    "Cape Verde": "Cabo Verde",
+}
+"""Affiliations EDMO Country name remapping dictionary."""
+
+VOCABULARIES_ORCID_ACCESS_KEY = "CHANGEME"
+"""ORCID access key to access the s3 bucket."""
+VOCABULARIES_ORCID_SECRET_KEY = "CHANGEME"
+"""ORCID secret key to access the s3 bucket."""
+VOCABULARIES_ORCID_SUMMARIES_BUCKET = "v3.0-summaries"
+"""ORCID summaries bucket name."""
+VOCABULARIES_ORCID_SYNC_MAX_WORKERS = 32
+"""ORCID max number of simultaneous workers/connections."""
+VOCABULARIES_ORCID_SYNC_SINCE = {
+    "days": 1,
+}
+"""ORCID time shift to sync. Parameters accepted are the ones passed to 'datetime.timedelta'."""
+
+VOCABULARIES_ORCID_ORG_IDS_MAPPING_PATH = None
+"""Path to the CSV file for mapping ORCiD organization IDs to affiliation IDs.
+
+The path can be specified as either an absolute path or a relative path within the
+Flask app instance folder (i.e. ``current_app.instance_path``).
+
+The CSV file should have the following columns:
+
+- `org_scheme`: The ORCiD organization ID.
+- `org_id`: The ORCiD organization ID.
+- `aff_id`: The affiliation ID to map to.
+"""

invenio_vocabularies/contrib/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Contrib module."""

invenio_vocabularies/contrib/affiliations/__init__.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Affiliations module."""
+
+from .resources import AffiliationsResource, AffiliationsResourceConfig
+from .services import AffiliationsService, AffiliationsServiceConfig
+
+__all__ = [
+    "AffiliationsResource",
+    "AffiliationsResourceConfig",
+    "AffiliationsService",
+    "AffiliationsServiceConfig",
+]

invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary affiliations."""
+
+from flask_resources import JSONSerializer, ResponseHandler
+from invenio_db import db
+from invenio_records.dumpers import SearchDumper
+from invenio_records.dumpers.indexedat import IndexedAtDumperExt
+from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.records.systemfields import ModelPIDField
+from invenio_records_resources.resources.records.headers import etag_headers
+
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
+from .config import AffiliationsSearchOptions, service_components
+from .schema import AffiliationSchema
+
+record_type = RecordTypeFactory(
+    "Affiliation",
+    # Data layer
+    pid_field_cls=ModelPIDField,
+    pid_field_kwargs={
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
+    },
+    schema_version="1.0.0",
+    schema_path="local://affiliations/affiliation-v1.0.0.json",
+    index_name="affiliations-affiliation-v2.0.0",
+    record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
+        extensions=[
+            IndexedAtDumperExt(),
+        ],
+    ),
+    # Service layer
+    service_id="affiliations",
+    service_schema=AffiliationSchema,
+    search_options=AffiliationsSearchOptions,
+    service_components=service_components,
+    permission_policy_cls=PermissionPolicy,
+    # Resource layer
+    endpoint_route="/affiliations",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
+)

invenio_vocabularies/contrib/affiliations/api.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary affiliations."""
+
+from .affiliations import record_type
+
+Affiliation = record_type.record_cls

invenio_vocabularies/contrib/affiliations/config.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary affiliations configuration."""
+
+from flask import current_app
+from invenio_i18n import get_locale
+from invenio_i18n import lazy_gettext as _
+from invenio_records_resources.services import SearchOptions
+from invenio_records_resources.services.records.components import DataComponent
+from invenio_records_resources.services.records.queryparser import (
+    CompositeSuggestQueryParser,
+)
+from werkzeug.local import LocalProxy
+
+from ...services.components import PIDComponent
+
+affiliation_schemes = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_AFFILIATION_SCHEMES"]
+)
+affiliation_edmo_country_mappings = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_AFFILIATIONS_EDMO_COUNTRY_MAPPING"]
+)
+localized_title = LocalProxy(lambda: f"title.{get_locale()}^2")
+
+
+class AffiliationsSearchOptions(SearchOptions):
+    """Search options."""
+
+    suggest_parser_cls = CompositeSuggestQueryParser.factory(
+        fields=[
+            # We boost the acronym fields, since they're smaller words and are more
+            # likely to be used in a query.
+            "acronym.keyword^50",
+            "acronym^10",
+            "name^10",
+            # Aliases can sometimes be shorter, so we boost them a bit.
+            "aliases^5",
+            localized_title,
+            # Allow to search identifiers directly (e.g. ROR)
+            "identifiers.identifier",
+            "country_name",
+        ],
+    )
+
+    sort_default = "bestmatch"
+
+    sort_default_no_query = "name"
+
+    sort_options = {
+        "bestmatch": dict(
+            title=_("Best match"),
+            fields=["_score"],  # ES defaults to desc on `_score` field
+        ),
+        "name": dict(
+            title=_("Name"),
+            fields=["name_sort"],
+        ),
+        "newest": dict(
+            title=_("Newest"),
+            fields=["-created"],
+        ),
+        "oldest": dict(
+            title=_("Oldest"),
+            fields=["created"],
+        ),
+    }
+
+
+service_components = [
+    # Order of components are important!
+    DataComponent,
+    PIDComponent,
+]