invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/datastreams/transformers.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Transformers module."""
+
+from abc import ABC, abstractmethod
+from urllib.parse import urlparse
+
+from lxml import etree
+
+from .errors import TransformerError
+from .xml import etree_to_dict
+
+try:
+    import rdflib
+except ImportError:
+    rdflib = None
+
+
+class BaseTransformer(ABC):
+    """Base transformer."""
+
+    @abstractmethod
+    def apply(self, stream_entry, *args, **kwargs):
+        """Applies the transformation to the entry.
+
+        :returns: A StreamEntry. The transformed entry.
+                  Raises TransformerError in case of errors.
+        """
+        pass
+
+
+class XMLTransformer(BaseTransformer):
+    """XML transformer."""
+
+    def __init__(self, root_element=None, *args, **kwargs):
+        """Initializes the transformer."""
+        self.root_element = root_element
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def _xml_to_etree(cls, xml):
+        """Converts XML to a lxml etree."""
+        return etree.HTML(xml)
+
+    def apply(self, stream_entry, **kwargs):
+        """Applies the transformation to the stream entry.
+
+        Requires the root element to be named "record".
+        """
+        xml_tree = self._xml_to_etree(stream_entry.entry)
+        xml_dict = etree_to_dict(xml_tree)["html"]["body"]
+
+        if self.root_element:
+            record = xml_dict.get(self.root_element)
+            if not record:
+                raise TransformerError(
+                    f"Root element '{self.root_element}' not found in XML entry."
+                )
+        else:
+            record = xml_dict
+
+        stream_entry.entry = record
+        return stream_entry
+
+
+class RDFTransformer(BaseTransformer):
+    """Base Transformer class for RDF data to dictionary format."""
+
+    @property
+    def skos_core(self):
+        """Get the SKOS core namespace."""
+        return rdflib.Namespace("http://www.w3.org/2004/02/skos/core#")
+
+    def _validate_subject_url(self, subject):
+        """Check if the subject is a valid URL."""
+        parsed = urlparse(str(subject))
+        return bool(parsed.netloc and parsed.scheme)
+
+    def _get_identifiers(self, subject):
+        """Generate identifiers field for a valid subject URL."""
+        if self._validate_subject_url(subject):
+            return [{"scheme": "url", "identifier": str(subject)}]
+        return []
+
+    def _get_labels(self, subject, rdf_graph):
+        """Extract labels (prefLabel or altLabel) for a subject."""
+        labels = {
+            label.language: label.value.capitalize()
+            for _, _, label in rdf_graph.triples(
+                (subject, self.skos_core.prefLabel, None)
+            )
+            if label.language and "-" not in label.language
+        }
+
+        if "en" not in labels:
+            for _, _, label in rdf_graph.triples(
+                (subject, self.skos_core.altLabel, None)
+            ):
+                labels.setdefault(label.language, label.value.capitalize())
+
+        return labels
+
+    def _find_parents(self, subject, rdf_graph):
+        """Find parent notations."""
+        return [
+            self._get_parent_notation(broader, rdf_graph)
+            for broader in rdf_graph.transitive_objects(subject, self.skos_core.broader)
+            if broader != subject
+        ]
+
+    def _get_parent_notation(self, broader, rdf_graph):
+        """Extract notation for a parent."""
+        raise NotImplementedError("This method should be implemented in a subclass.")
+
+    def _transform_entry(self, subject, rdf_graph):
+        """Transform an RDF subject entry into the desired dictionary format."""
+        raise NotImplementedError("This method should be implemented in a subclass.")
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Apply transformation to a stream entry."""
+        stream_entry.entry = self._transform_entry(
+            stream_entry.entry["subject"], stream_entry.entry["rdf_graph"]
+        )
+        return stream_entry

invenio_vocabularies/datastreams/writers.py

@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Writers module."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+import yaml
+from flask import current_app
+from invenio_access.permissions import system_identity
+from invenio_pidstore.errors import PIDAlreadyExists, PIDDoesNotExistError
+from invenio_records.systemfields.relations.errors import InvalidRelationValue
+from invenio_records_resources.proxies import current_service_registry
+from marshmallow import ValidationError
+from sqlalchemy.exc import NoResultFound
+
+from .datastreams import StreamEntry
+from .errors import WriterError
+from .tasks import write_entry, write_many_entry
+
+
+class BaseWriter(ABC):
+    """Base writer."""
+
+    is_async = False
+
+    def __init__(self, *args, **kwargs):
+        """Base initialization logic."""
+        # Add any base initialization here if needed
+        pass
+
+    @abstractmethod
+    def write(self, stream_entry, *args, **kwargs):
+        """Writes the input stream entry to the target output.
+
+        :returns: A StreamEntry. The result of writing the entry.
+                  Raises WriterException in case of errors.
+
+        """
+        pass
+
+    @abstractmethod
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input streams entry to the target output.
+
+        :returns: A List of StreamEntry. The result of writing the entry.
+                  Raises WriterException in case of errors.
+
+        """
+        pass
+
+
+class ServiceWriter(BaseWriter):
+    """Writes the entries to an RDM instance using a Service object."""
+
+    def __init__(
+        self, service_or_name, *args, identity=None, insert=True, update=False, **kwargs
+    ):
+        """Constructor.
+
+        :param service_or_name: a service instance or a key of the
+                                service registry.
+        :param identity: access identity.
+        :param insert: if True it will insert records which do not exist.
+        :param update: if True it will update records if they exist.
+        """
+        if isinstance(service_or_name, str):
+            service_or_name = current_service_registry.get(service_or_name)
+
+        self._service = service_or_name
+        self._identity = identity or system_identity
+        self._insert = insert
+        self._update = update
+
+        super().__init__(*args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return (entry["type"], entry["id"])
+
+    def _resolve(self, id_):
+        return self._service.read(self._identity, id_)
+
+    def _do_update(self, entry):
+        vocab_id = self._entry_id(entry)
+        current_app.logger.debug(f"Resolving entry with ID: {vocab_id}")
+        current = self._resolve(vocab_id)
+        updated = dict(current.to_dict(), **entry)
+        current_app.logger.debug(f"Updating entry with ID: {vocab_id}")
+        return StreamEntry(
+            self._service.update(self._identity, vocab_id, updated), op_type="update"
+        )
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Writes the input entry using a given service."""
+        entry = stream_entry.entry
+        current_app.logger.debug(f"Writing entry: {entry}")
+
+        try:
+            if self._insert:
+                try:
+                    current_app.logger.debug("Inserting entry.")
+                    return StreamEntry(
+                        self._service.create(self._identity, entry), op_type="create"
+                    )
+                except PIDAlreadyExists:
+                    if not self._update:
+                        raise WriterError([f"Vocabulary entry already exists: {entry}"])
+                    return self._do_update(entry)
+            elif self._update:
+                try:
+                    current_app.logger.debug("Attempting to update entry.")
+                    return self._do_update(entry)
+                except (NoResultFound, PIDDoesNotExistError):
+                    raise WriterError([f"Vocabulary entry does not exist: {entry}"])
+            else:
+                raise WriterError(
+                    ["Writer wrongly configured to not insert and to not update"]
+                )
+
+        except ValidationError as err:
+            raise WriterError([{"ValidationError": err.messages}])
+        except InvalidRelationValue as err:
+            # TODO: Check if we can get the error message easier
+            raise WriterError([{"InvalidRelationValue": err.args[0]}])
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input entries using a given service."""
+        current_app.logger.info(f"Writing {len(stream_entries)} entries")
+        entries = [entry.entry for entry in stream_entries]
+        entries_with_id = [(self._entry_id(entry), entry) for entry in entries]
+        result_list = self._service.create_or_update_many(
+            self._identity, entries_with_id
+        )
+        stream_entries_processed = []
+        for entry, result in zip(entries, result_list.results):
+            processed_stream_entry = StreamEntry(
+                entry=entry,
+                record=result.record,
+                errors=result.errors,
+                op_type=result.op_type,
+                exc=result.exc,
+            )
+            processed_stream_entry.log_errors()
+            stream_entries_processed.append(processed_stream_entry)
+
+        current_app.logger.debug(f"Finished writing {len(stream_entries)} entries")
+        return stream_entries_processed
+
+
+class YamlWriter(BaseWriter):
+    """Writes the entries to a YAML file."""
+
+    def __init__(self, filepath, *args, **kwargs):
+        """Constructor.
+
+        :param filepath: path of the output file.
+        """
+        self._filepath = Path(filepath)
+
+        super().__init__(*args, **kwargs)
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Writes the input stream entry using a given service."""
+        with open(self._filepath, "a") as file:
+            # made into array for safer append
+            # will always read array (good for reader)
+            yaml.safe_dump([stream_entry.entry], file, allow_unicode=True)
+
+        return stream_entry
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the yaml input entries."""
+        with open(self._filepath, "a") as file:
+            yaml.safe_dump(
+                [stream_entry.entry for stream_entry in stream_entries],
+                file,
+                allow_unicode=True,
+            )
+
+
+class AsyncWriter(BaseWriter):
+    """Writes the entries asynchronously (celery task)."""
+
+    is_async = True
+
+    def __init__(self, writer, *args, **kwargs):
+        """Constructor.
+
+        :param writer: writer to use.
+        """
+        super().__init__(*args, **kwargs)
+        self._writer = writer
+
+    def write(self, stream_entry, subtask_run_id=None, *args, **kwargs):
+        """Launches a celery task to write an entry with a delay."""
+        # Add some delay to avoid processing the tasks too fast
+        write_entry.apply_async(
+            args=(self._writer, stream_entry.entry, subtask_run_id), countdown=1
+        )
+
+        return stream_entry
+
+    def write_many(self, stream_entries, subtask_run_id=None, *args, **kwargs):
+        """Launches a celery task to write entries with a delay."""
+        # Add some delay to avoid processing the tasks too fast
+        write_many_entry.apply_async(
+            args=(
+                self._writer,
+                [stream_entry.entry for stream_entry in stream_entries],
+                subtask_run_id,
+            ),
+            countdown=1,
+        )
+
+        return stream_entries

invenio_vocabularies/datastreams/xml.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""XML utils."""
+
+from collections import defaultdict
+
+
+def etree_to_dict(tree):
+    """Convert an ElementTree to a dictionary."""
+    tag = tree.tag.split(":")[-1]  # strip namespace
+    d = {tag: {} if tree.attrib else None}
+    children = list(tree)
+    if children:
+        dd = defaultdict(list)
+        for dc in map(etree_to_dict, children):
+            for k, v in dc.items():
+                dd[k].append(v)
+        d = {tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
+    if tree.attrib:
+        d[tag].update(("@" + k, v) for k, v in tree.attrib.items())
+    if tree.text:
+        text = tree.text.strip()
+        if children or tree.attrib:
+            if text:
+                d[tag]["#text"] = text
+        else:
+            d[tag] = text
+    return d

invenio_vocabularies/ext.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2023 Graz University of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Invenio module for managing vocabularies."""
+
+from . import config
+from .contrib.affiliations import (
+    AffiliationsResource,
+    AffiliationsResourceConfig,
+    AffiliationsService,
+    AffiliationsServiceConfig,
+)
+from .contrib.awards import (
+    AwardsResource,
+    AwardsResourceConfig,
+    AwardsService,
+    AwardsServiceConfig,
+)
+from .contrib.funders import (
+    FundersResource,
+    FundersResourceConfig,
+    FundersService,
+    FundersServiceConfig,
+)
+from .contrib.names import (
+    NamesResource,
+    NamesResourceConfig,
+    NamesService,
+    NamesServiceConfig,
+)
+from .contrib.subjects import (
+    SubjectsResource,
+    SubjectsResourceConfig,
+    SubjectsService,
+    SubjectsServiceConfig,
+)
+from .resources import (
+    VocabulariesAdminResource,
+    VocabulariesResource,
+    VocabulariesResourceConfig,
+    VocabularyTypeResourceConfig,
+)
+from .services.config import VocabularyTypesServiceConfig
+from .services.service import VocabulariesService, VocabularyTypeService
+
+
+class InvenioVocabularies(object):
+    """Invenio-Vocabularies extension."""
+
+    def __init__(self, app=None):
+        """Extension initialization."""
+        self.resource = None
+        self.service = None
+        if app:
+            self.init_app(app)
+
+    def init_app(self, app):
+        """Flask application initialization."""
+        self.init_config(app)
+        self.init_services(app)
+        self.init_resource(app)
+        app.extensions["invenio-vocabularies"] = self
+
+    def init_config(self, app):
+        """Initialize configuration."""
+        for k in dir(config):
+            if k.startswith("VOCABULARIES_"):
+                app.config.setdefault(k, getattr(config, k))
+
+    def service_configs(self, app):
+        """Customized service configs."""
+
+        class ServiceConfigs:
+            affiliations = AffiliationsServiceConfig
+            awards = AwardsServiceConfig
+            funders = FundersServiceConfig
+            names = NamesServiceConfig
+            subjects = SubjectsServiceConfig
+            vocabulary_types = VocabularyTypesServiceConfig
+
+        return ServiceConfigs
+
+    def init_services(self, app):
+        """Initialize vocabulary resources."""
+        service_configs = self.service_configs(app)
+
+        # Services
+        self.affiliations_service = AffiliationsService(
+            config=service_configs.affiliations,
+        )
+        self.awards_service = AwardsService(
+            config=service_configs.awards,
+        )
+        self.funders_service = FundersService(config=service_configs.funders)
+        self.names_service = NamesService(config=service_configs.names)
+        self.subjects_service = SubjectsService(config=service_configs.subjects)
+        self.vocabularies_service = VocabulariesService(
+            config=app.config["VOCABULARIES_SERVICE_CONFIG"],
+        )
+        self.vocabulary_types_service = VocabularyTypeService(
+            config=service_configs.vocabulary_types
+        )
+
+    def init_resource(self, app):
+        """Initialize vocabulary resources."""
+        # Generic Vocabularies
+        self.affiliations_resource = AffiliationsResource(
+            service=self.affiliations_service,
+            config=AffiliationsResourceConfig,
+        )
+        self.funders_resource = FundersResource(
+            service=self.funders_service,
+            config=FundersResourceConfig,
+        )
+        self.names_resource = NamesResource(
+            service=self.names_service,
+            config=NamesResourceConfig,
+        )
+        self.awards_resource = AwardsResource(
+            service=self.awards_service,
+            config=AwardsResourceConfig,
+        )
+        self.subjects_resource = SubjectsResource(
+            service=self.subjects_service,
+            config=SubjectsResourceConfig,
+        )
+        self.resource = VocabulariesResource(
+            service=self.vocabularies_service,
+            config=app.config["VOCABULARIES_RESOURCE_CONFIG"],
+        )
+        self.vocabulary_admin_resource = VocabulariesAdminResource(
+            service=self.vocabulary_types_service,
+            config=VocabularyTypeResourceConfig,
+        )
+
+
+def finalize_app(app):
+    """Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def api_finalize_app(app):
+    """Api Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def init(app):
+    """Init app."""
+    # Register services - cannot be done in extension because
+    # Invenio-Records-Resources might not have been initialized.
+    sregistry = app.extensions["invenio-records-resources"].registry
+    ext = app.extensions["invenio-vocabularies"]
+    sregistry.register(ext.affiliations_service, service_id="affiliations")
+    sregistry.register(ext.awards_service, service_id="awards")
+    sregistry.register(ext.funders_service, service_id="funders")
+    sregistry.register(ext.names_service, service_id="names")
+    sregistry.register(ext.subjects_service, service_id="subjects")
+    sregistry.register(ext.vocabularies_service, service_id="vocabularies")
+    sregistry.register(ext.vocabulary_types_service, service_id="vocabulary-types")
+    # Register indexers
+    iregistry = app.extensions["invenio-indexer"].registry
+    iregistry.register(ext.affiliations_service.indexer, indexer_id="affiliations")
+    iregistry.register(ext.awards_service.indexer, indexer_id="awards")
+    iregistry.register(ext.funders_service.indexer, indexer_id="funders")
+    iregistry.register(ext.names_service.indexer, indexer_id="names")
+    iregistry.register(ext.subjects_service.indexer, indexer_id="subjects")
+    iregistry.register(ext.vocabularies_service.indexer, indexer_id="vocabularies")