invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/contrib/subjects/nvs/datastreams.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024-2025 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""NVS subjects datastreams, readers, transformers, and writers."""
+
+from invenio_vocabularies.datastreams.errors import TransformerError
+from invenio_vocabularies.datastreams.readers import RDFReader
+from invenio_vocabularies.datastreams.transformers import RDFTransformer
+
+from ..config import nvs_file_url
+
+# Available with the "rdf" extra
+try:
+    import rdflib
+except ImportError:
+    rdflib = None
+
+
+class NVSSubjectsTransformer(RDFTransformer):
+    """
+    Transformer class to convert NVS RDF data to a dictionary format.
+
+    Input:
+        - Relevant fields:
+            - `skos:notation`: Primary identifier for the concept.
+            - `skos:prefLabel`: Preferred labels with language codes.
+            - `skos:altLabel`: Alternative labels (optional).
+            - `skos:definition`: Definitions of the concept.
+            - `owl:deprecated`: Boolean flag indicating if the concept is deprecated.
+
+    Output:
+        - A dictionary with the following structure:
+            {
+                "id": "SDN:P01::SAGEMSFM",  # NVS-specific parameter ID (skos:notation).
+                "scheme": "NVS-P01",  # The scheme name indicating this is a collection P01 from NERC Vocabulary Server (NVS).
+                "subject": "AMSSedAge",  # The alternative label (skos:altLabel), if available, or None.
+                "title": {
+                    "en": "14C age of Foraminiferida"  # English preferred label (skos:prefLabel).
+                },
+                "props": {
+                    "definitions": "Accelerated mass spectrometry on picked tests",  # Definition of subject (skos:definition).
+                },
+                "identifiers": [
+                    {
+                        "scheme": "url",  # Type of identifier (URL).
+                        "identifier": "http://vocab.nerc.ac.uk/collection/P01/current/SAGEMSFM"  # URI of the concept.
+                    }
+                ]
+            }
+    """
+
+    def _get_subject_data(self, rdf_graph, subject):
+        """Fetch all triples for a subject and organize them into a dictionary."""
+        data = {}
+        for predicate, obj in rdf_graph.predicate_objects(subject=subject):
+            predicate_name = str(predicate)
+            if predicate_name not in data:
+                data[predicate_name] = []
+            data[predicate_name].append(obj)
+        return data
+
+    def _transform_entry(self, subject, rdf_graph):
+        """Transform an entry to the required dictionary format."""
+        labels = self._get_labels(subject, rdf_graph)
+        subject_data = self._get_subject_data(rdf_graph, subject)
+        deprecated = subject_data.get(str(rdflib.namespace.OWL.deprecated), [False])
+        if deprecated and str(deprecated[0]).lower() == "true":
+            raise TransformerError(f"Skipping deprecated subject: {subject_data}")
+
+        notation = subject_data.get(str(self.skos_core.notation), [])
+        if notation:
+            id = str(notation[0])
+        else:
+            raise TransformerError(f"No id found for: {subject}")
+
+        pref_labels = [
+            obj for obj in subject_data.get(str(self.skos_core.prefLabel), [])
+        ]
+
+        subject_text = str(pref_labels[0]) if pref_labels else labels["en"]
+        definition = str(subject_data.get(str(self.skos_core.definition), [None])[0])
+
+        return {
+            "id": id,
+            "scheme": "NVS-P02",
+            "subject": subject_text,
+            "title": labels,
+            "props": {"definition": definition} if definition else {},
+            "identifiers": self._get_identifiers(subject),
+        }
+
+
+# Configuration for datastream
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"nvs-transformer": NVSSubjectsTransformer}
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "http",
+            "args": {
+                "origin": nvs_file_url,
+            },
+        },
+        {"type": "rdf"},
+    ],
+    "transformers": [{"type": "nvs-transformer"}],
+    "writers": [{"args": {"writer": {"type": "subjects-service"}}, "type": "async"}],
+}

invenio_vocabularies/contrib/subjects/resources.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Test the affiliation vocabulary resources."""
+
+from .subjects import record_type
+
+SubjectsResourceConfig = record_type.resource_config_cls
+SubjectsResourceConfig.routes["item"] = "/<path:pid_value>"
+
+SubjectsResource = record_type.resource_cls

invenio_vocabularies/contrib/subjects/schema.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C) 2024 University of Münster.
+# Copyright (C) 2025 Graz University of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Subjects schema."""
+
+from functools import partial
+
+from invenio_i18n import get_locale
+from marshmallow import EXCLUDE, Schema, ValidationError, fields, pre_load, validate
+from marshmallow_utils.fields import URL, IdentifierSet, SanitizedUnicode
+from marshmallow_utils.schemas import IdentifierSchema
+
+from ...services.schema import (
+    BaseVocabularySchema,
+    ContribVocabularyRelationSchema,
+    i18n_strings,
+)
+from .config import subject_schemes
+
+
+class StringOrListOfStrings(fields.Raw):
+    """Custom field to handle both string and list of strings."""
+
+    # TODO: Move this to marshmallow-utils for broader type support.
+    def _deserialize(self, value, attr, data, **kwargs):
+        if isinstance(value, str):
+            return fields.String()._deserialize(value, attr, data, **kwargs)
+        elif isinstance(value, list) and all(isinstance(item, str) for item in value):
+            return [
+                fields.String()._deserialize(item, attr, data, **kwargs)
+                for item in value
+            ]
+        raise ValidationError("Invalid value. Must be a string or a list of strings.")
+
+
+class SubjectSchema(BaseVocabularySchema):
+    """Service schema for subjects."""
+
+    # id in BaseRecordSchema is not required, but I don't see why it shouldn't
+    # be, while scheme and subject are required. So I am making it required
+    # here.
+    id = SanitizedUnicode(required=True)
+    scheme = SanitizedUnicode(required=True)
+    subject = SanitizedUnicode(required=True)
+    title = i18n_strings
+    props = fields.Dict(keys=SanitizedUnicode(), values=StringOrListOfStrings())
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode())
+
+    @pre_load
+    def add_subject_from_title(self, data, **kwargs):
+        """Add subject from title if not present."""
+        locale = get_locale().language
+        if "subject" not in data:
+            data["subject"] = data["title"].get(locale) or data["title"].values()[0]
+        return data
+
+
+class SubjectRelationSchema(ContribVocabularyRelationSchema):
+    """Schema to define an optional subject relation in another schema."""
+
+    # If re-running an OpenAIRE awards update on existing awards which already have subjects,
+    # the subject entries will contains `scheme` and `props`, which are unknown since they are `dump_only`.
+    # This makes the update exclude unknown field and go through with the update.
+    class Meta:
+        """Metadata class."""
+
+        unknown = EXCLUDE
+
+    ftf_name = "subject"
+    parent_field_name = "subjects"
+    subject = SanitizedUnicode()
+    scheme = SanitizedUnicode(dump_only=True)
+    title = fields.Dict(dump_only=True)
+    props = fields.Dict(dump_only=True)
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode(), dump_only=True)

invenio_vocabularies/contrib/subjects/services.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Subjects services."""
+
+from invenio_db import db
+
+from ...records.models import VocabularyScheme
+from .subjects import record_type
+
+SubjectsServiceConfig = record_type.service_config_cls
+
+
+class SubjectsService(record_type.service_cls):
+    """Subjects service."""
+
+    def create_scheme(self, identity, id_, name="", uri=""):
+        """Create a row for the subject scheme metadata."""
+        self.require_permission(identity, "manage")
+        scheme = VocabularyScheme.create(
+            id=id_, parent_id="subjects", name=name, uri=uri
+        )
+        db.session.commit()
+        return scheme

invenio_vocabularies/contrib/subjects/subjects.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary subjects."""
+
+from flask_resources import JSONSerializer, ResponseHandler
+from invenio_records.dumpers import SearchDumper
+from invenio_records.dumpers.indexedat import IndexedAtDumperExt
+from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.resources.records.headers import etag_headers
+
+from ...records.pidprovider import PIDProviderFactory
+from ...records.systemfields import BaseVocabularyPIDFieldContext
+from ...services.permissions import PermissionPolicy
+from .config import SubjectsSearchOptions, service_components
+from .schema import SubjectSchema
+
+record_type = RecordTypeFactory(
+    "Subject",
+    # Data layer
+    pid_field_kwargs={
+        "create": False,
+        "provider": PIDProviderFactory.create(pid_type="sub"),
+        "context_cls": BaseVocabularyPIDFieldContext,
+    },
+    schema_version="1.0.0",
+    schema_path="local://subjects/subject-v1.0.0.json",
+    record_dumper=SearchDumper(
+        extensions=[
+            IndexedAtDumperExt(),
+        ]
+    ),
+    # Service layer
+    service_id="subjects",
+    service_schema=SubjectSchema,
+    search_options=SubjectsSearchOptions,
+    service_components=service_components,
+    permission_policy_cls=PermissionPolicy,
+    # Resource layer
+    endpoint_route="/subjects",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
+)

invenio_vocabularies/datastreams/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Datastreams module."""
+
+from .datastreams import DataStream, StreamEntry
+from .factories import DataStreamFactory
+
+__all__ = (
+    "DataStream",
+    "DataStreamFactory",
+    "StreamEntry",
+)

invenio_vocabularies/datastreams/datastreams.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Base data stream."""
+
+from flask import current_app
+from invenio_access.permissions import system_identity, system_user_id
+from invenio_access.utils import get_identity
+from invenio_accounts.proxies import current_datastore
+from invenio_jobs.logging.jobs import EMPTY_JOB_CTX, job_context
+from invenio_jobs.proxies import current_runs_service
+
+from .errors import ReaderError, TransformerError, WriterError
+
+
+class StreamEntry:
+    """Object to encapsulate streams processing."""
+
+    def __init__(self, entry, record=None, errors=None, op_type=None, exc=None):
+        """Constructor for the StreamEntry class.
+
+        :param entry (object): The entry object, usually a record dict.
+        :param record (object): The record object, usually a record class.
+        :param errors (list, optional): List of errors. Defaults to None.
+        :param op_type (str, optional): The operation type. Defaults to None.
+        :param exc (str, optional): The raised unhandled exception. Defaults to None.
+        """
+        self.entry = entry
+        self.record = record
+        self.filtered = False
+        self.errors = errors or []
+        self.op_type = op_type
+        self.exc = exc
+
+    def log_errors(self, logger=None):
+        """Log the errors using the provided logger or the default logger.
+
+        :param logger (logging.Logger, optional): Logger instance to use. Defaults to None.
+        """
+        if logger is None:
+            logger = current_app.logger
+        for error in self.errors:
+            logger.error(f"Error in entry {self.entry}: {error}")
+        if self.exc:
+            logger.error(f"Exception in entry {self.entry}: {self.exc}")
+
+
+class DataStream:
+    """Data stream."""
+
+    def __init__(
+        self,
+        readers,
+        writers,
+        transformers=None,
+        batch_size=100,
+        write_many=False,
+        *args,
+        **kwargs,
+    ):
+        """Constructor.
+
+        :param readers: an ordered list of readers.
+        :param writers: an ordered list of writers.
+        :param transformers: an ordered list of transformers to apply.
+        """
+        self._readers = readers
+        self._transformers = transformers
+        self._writers = writers
+        self.batch_size = batch_size
+        self.write_many = write_many
+
+    def filter(self, stream_entry, *args, **kwargs):
+        """Checks if an stream_entry should be filtered out (skipped)."""
+        current_app.logger.debug(f"Filtering entry: {stream_entry.entry}")
+        return False
+
+    def process_batch(self, batch):
+        """Process a batch of entries."""
+        current_app.logger.info(f"Processing batch of size: {len(batch)}")
+        if job_context.get() is not EMPTY_JOB_CTX:
+            run_id = job_context.get()["run_id"]
+            current_runs_service.add_total_entries(
+                system_identity,
+                run_id=run_id,
+                job_id=job_context.get()["job_id"],
+                total_entries=len(batch),
+            )
+        transformed_entries = []
+        transformed_entries_with_errors = []
+        for stream_entry in batch:
+            if stream_entry.errors:
+                current_app.logger.warning(
+                    f"Skipping entry with errors: {stream_entry.errors}"
+                )
+                yield stream_entry  # reading errors
+            else:
+                transformed_entry = self.transform(stream_entry)
+                if transformed_entry.errors:
+                    transformed_entries_with_errors.append(transformed_entry)
+                    yield transformed_entry
+                elif self.filter(transformed_entry):
+                    transformed_entry.filtered = True
+                    yield transformed_entry
+                else:
+                    transformed_entries.append(transformed_entry)
+        if transformed_entries_with_errors:
+            current_app.logger.warning(
+                f"Skipping {len(transformed_entries_with_errors)} transformed entries with errors."
+            )
+        if transformed_entries:
+            if self.write_many:
+                yield from self.batch_write(transformed_entries)
+            else:
+                yield from (self.write(entry) for entry in transformed_entries)
+
+    def process(self, *args, **kwargs):
+        """Iterates over the entries.
+
+        Uses the reader to get the raw entries and transforms them.
+        It will iterate over the `StreamEntry` objects returned by
+        the reader, apply the transformations and yield the result of
+        writing it.
+        """
+        current_app.logger.info("Starting data stream processing")
+        batch = []
+        for stream_entry in self.read():
+            batch.append(stream_entry)
+            if len(batch) >= self.batch_size:
+                current_app.logger.debug(f"Processing batch of size: {len(batch)}")
+                yield from self.process_batch(batch)
+                batch = []
+
+        # Process any remaining entries in the last batch
+        if batch:
+            current_app.logger.debug(f"Processing final batch of size: {len(batch)}")
+            yield from self.process_batch(batch)
+
+    def read(self):
+        """Recursively read the entries."""
+        current_app.logger.debug("Reading entries from readers")
+
+        def pipe_gen(gen_funcs, piped_item=None):
+            _gen_funcs = list(gen_funcs)  # copy to avoid modifying ref list
+            # use and remove the current generator
+            current_gen_func = _gen_funcs.pop(0)
+            for item in current_gen_func(piped_item):
+                try:
+                    # exhaust iterations of subsequent generators
+                    if _gen_funcs:
+                        yield from pipe_gen(_gen_funcs, piped_item=item)
+                    # there is no subsequent generator, return the current item
+                    else:
+                        yield StreamEntry(item)
+                except ReaderError as err:
+                    current_app.logger.error(f"Reader error: {str(err)}")
+                    yield StreamEntry(
+                        entry=item,
+                        errors=[f"{current_gen_func.__qualname__}: {str(err)}"],
+                    )
+
+        read_gens = [r.read for r in self._readers]
+        yield from pipe_gen(read_gens)
+
+    def transform(self, stream_entry, *args, **kwargs):
+        """Apply the transformations to an stream_entry."""
+        current_app.logger.debug(f"Transforming entry: {stream_entry.entry}")
+        for transformer in self._transformers:
+            try:
+                stream_entry = transformer.apply(stream_entry)
+            except TransformerError as err:
+                stream_entry.errors.append(
+                    f"{transformer.__class__.__name__}: {str(err)}"
+                )
+                return stream_entry  # break loop
+
+        return stream_entry
+
+    def _prepare_async_context(self):
+        """Prepare the async context for writers."""
+        job_ctx = job_context.get()
+        run_id = job_ctx.get("run_id")
+        identity_id = job_ctx.get("identity_id")
+        job_id = job_ctx.get("job_id")
+        # System user needs to be handled separately because it doesn't exist in the database
+        if identity_id == system_user_id:
+            identity = system_identity
+        else:
+            user = current_datastore.get_user(identity_id)
+            if user is None:
+                raise ValueError(f"User with ID:{identity_id} not found.")
+            identity = get_identity(user)
+
+        subtask_run = current_runs_service.create_subtask_run(
+            identity, parent_run_id=run_id, job_id=job_id
+        )
+        return str(subtask_run.id)
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Write a single stream entry."""
+        current_app.logger.debug(f"Writing entry: {stream_entry.entry}")
+        for writer in self._writers:
+            try:
+                if writer.is_async and job_context.get() is not EMPTY_JOB_CTX:
+                    subtask_run_id = self._prepare_async_context()
+                    writer.write(stream_entry, subtask_run_id=subtask_run_id)
+                else:
+                    writer.write(stream_entry)
+            except WriterError as err:
+                current_app.logger.error(f"Writer error: {str(err)}")
+                stream_entry.errors.append(f"{writer.__class__.__name__}: {str(err)}")
+
+        return stream_entry
+
+    def batch_write(self, stream_entries, *args, **kwargs):
+        """Write a batch of stream entries."""
+        current_app.logger.debug(f"Batch writing entries: {len(stream_entries)}")
+        for writer in self._writers:
+            try:
+                if writer.is_async and job_context.get() is not EMPTY_JOB_CTX:
+                    subtask_run_id = self._prepare_async_context()
+                    yield from writer.write_many(
+                        stream_entries, subtask_run_id=subtask_run_id
+                    )
+                else:
+                    yield from writer.write_many(stream_entries)
+            except WriterError as err:
+                current_app.logger.error(f"Writer error: {str(err)}")
+                for entry in stream_entries:
+                    entry.errors.append(f"{writer.__class__.__name__}: {str(err)}")
+
+    def total(self, *args, **kwargs):
+        """The total of entries obtained from the origin."""
+        raise NotImplementedError()

invenio_vocabularies/datastreams/errors.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Datastream errors."""
+
+
+class ReaderError(Exception):
+    """Transformer application exception."""
+
+
+class TransformerError(Exception):
+    """Transformer application exception."""
+
+
+class WriterError(Exception):
+    """Transformer application exception."""
+
+
+class FactoryError(Exception):
+    """Transformer application exception."""
+
+    def __init__(self, name, key):
+        """Initialise error."""
+        super().__init__(f"{name} {key} not configured.")