invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/subjects/mesh/datastreams.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH subjects datastreams, transformers, writers and readers."""
+
+from invenio_vocabularies.datastreams.transformers import (
+    BaseTransformer,
+    TransformerError,
+)
+
+
+class MeshSubjectsTransformer(BaseTransformer):
+    """MeSH subjects Transformer."""
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Apply transformation on steam entry."""
+        entry_data = stream_entry.entry
+
+        # ID in MeSH data is the URL, ex. https://id.nlm.nih.gov/mesh/D000001
+        # We just want to use the ID prefixed by "mesh:""
+        try:
+            mesh_id = entry_data["id"].split("/")[-1]
+        except Exception:
+            raise TransformerError("Not a valid MeSH ID.")
+
+        entry_data["id"] = "mesh:" + mesh_id
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""MeSH datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+"""MeSH subject datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"mesh-subjects": MeshSubjectsTransformer}
+"""MeSH subjects datastream transformers."""

invenio_vocabularies/contrib/subjects/schema.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021 Northwestern University.
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,10 +10,19 @@
 
 """Subjects schema."""
 
-from invenio_i18n import lazy_gettext as _
-from marshmallow_utils.fields import SanitizedUnicode
+from functools import partial
 
-from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
+from invenio_i18n import get_locale
+from marshmallow import Schema, fields, pre_load
+from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
+from marshmallow_utils.schemas import IdentifierSchema
+
+from ...services.schema import (
+    BaseVocabularySchema,
+    ContribVocabularyRelationSchema,
+    i18n_strings,
+)
+from .config import subject_schemes
 
 
 class SubjectSchema(BaseVocabularySchema):
@@ -24,6 +34,26 @@ class SubjectSchema(BaseVocabularySchema):
     id = SanitizedUnicode(required=True)
     scheme = SanitizedUnicode(required=True)
     subject = SanitizedUnicode(required=True)
+    title = i18n_strings
+    props = fields.Dict(keys=SanitizedUnicode(), values=SanitizedUnicode())
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode())
+
+    @pre_load
+    def add_subject_from_title(self, data, **kwargs):
+        """Add subject from title if not present."""
+        locale = get_locale().language
+        if "subject" not in data:
+            data["subject"] = data["title"].get(locale) or data["title"].values()[0]
+        return data
 
 
 class SubjectRelationSchema(ContribVocabularyRelationSchema):
@@ -32,4 +62,16 @@ class SubjectRelationSchema(ContribVocabularyRelationSchema):
     ftf_name = "subject"
     parent_field_name = "subjects"
     subject = SanitizedUnicode()
-    scheme = SanitizedUnicode()
+    scheme = SanitizedUnicode(dump_only=True)
+    title = fields.Dict(dump_only=True)
+    props = fields.Dict(dump_only=True)
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode(), dump_only=True)

invenio_vocabularies/contrib/subjects/subjects.py

@@ -9,9 +9,11 @@
 
 """Vocabulary subjects."""
 
+from flask_resources import JSONSerializer, ResponseHandler
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...records.pidprovider import PIDProviderFactory
 from ...records.systemfields import BaseVocabularyPIDFieldContext
@@ -42,4 +44,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/subjects",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

invenio_vocabularies/datastreams/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,17 +8,41 @@
 
 """Base data stream."""
 
+from flask import current_app
+
 from .errors import ReaderError, TransformerError, WriterError
 
 
 class StreamEntry:
     """Object to encapsulate streams processing."""
 
-    def __init__(self, entry, errors=None):
-        """Constructor."""
+    def __init__(self, entry, record=None, errors=None, op_type=None, exc=None):
+        """Constructor for the StreamEntry class.
+
+        :param entry (object): The entry object, usually a record dict.
+        :param record (object): The record object, usually a record class.
+        :param errors (list, optional): List of errors. Defaults to None.
+        :param op_type (str, optional): The operation type. Defaults to None.
+        :param exc (str, optional): The raised unhandled exception. Defaults to None.
+        """
         self.entry = entry
+        self.record = record
         self.filtered = False
         self.errors = errors or []
+        self.op_type = op_type
+        self.exc = exc
+
+    def log_errors(self, logger=None):
+        """Log the errors using the provided logger or the default logger.
+
+        :param logger (logging.Logger, optional): Logger instance to use. Defaults to None.
+        """
+        if logger is None:
+            logger = current_app.logger
+        for error in self.errors:
+            logger.error(f"Error in entry {self.entry}: {error}")
+        if self.exc:
+            logger.error(f"Exception in entry {self.entry}: {self.exc}")
 
 
 class DataStream:
@@ -39,15 +63,10 @@ class DataStream:
         """Checks if an stream_entry should be filtered out (skipped)."""
         return False
 
-    def process(self, *args, **kwargs):
-        """Iterates over the entries.
-
-        Uses the reader to get the raw entries and transforms them.
-        It will iterate over the `StreamEntry` objects returned by
-        the reader, apply the transformations and yield the result of
-        writing it.
-        """
-        for stream_entry in self.read():
+    def process_batch(self, batch, write_many=False):
+        """Process a batch of entries."""
+        transformed_entries = []
+        for stream_entry in batch:
             if stream_entry.errors:
                 yield stream_entry  # reading errors
             else:
@@ -58,7 +77,31 @@ class DataStream:
                     transformed_entry.filtered = True
                     yield transformed_entry
                 else:
-                    yield self.write(transformed_entry)
+                    transformed_entries.append(transformed_entry)
+        if transformed_entries:
+            if write_many:
+                yield from self.batch_write(transformed_entries)
+            else:
+                yield from (self.write(entry) for entry in transformed_entries)
+
+    def process(self, batch_size=100, write_many=False, *args, **kwargs):
+        """Iterates over the entries.
+
+        Uses the reader to get the raw entries and transforms them.
+        It will iterate over the `StreamEntry` objects returned by
+        the reader, apply the transformations and yield the result of
+        writing it.
+        """
+        batch = []
+        for stream_entry in self.read():
+            batch.append(stream_entry)
+            if len(batch) >= batch_size:
+                yield from self.process_batch(batch, write_many=write_many)
+                batch = []
+
+        # Process any remaining entries in the last batch
+        if batch:
+            yield from self.process_batch(batch, write_many=write_many)
 
     def read(self):
         """Recursively read the entries."""
@@ -107,6 +150,11 @@ class DataStream:
 
         return stream_entry
 
+    def batch_write(self, stream_entries, *args, **kwargs):
+        """Apply the transformations to an stream_entry. Errors are handler in the service layer."""
+        for writer in self._writers:
+            yield from writer.write_many(stream_entries)
+
     def total(self, *args, **kwargs):
         """The total of entries obtained from the origin."""
         raise NotImplementedError()

invenio_vocabularies/datastreams/factories.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -36,7 +36,6 @@ class Factory:
         try:
             type_ = config["type"]
             args = config.get("args", {})
-
             return cls.options()[type_](**args)
         except KeyError:
             raise FactoryError(name=cls.FACTORY_NAME, key=type_)

invenio_vocabularies/datastreams/readers.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C)      2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -15,14 +16,21 @@ import re
 import tarfile
 import zipfile
 from abc import ABC, abstractmethod
-from collections import defaultdict
 from json.decoder import JSONDecodeError
 
 import requests
 import yaml
+from lxml import etree
+from lxml.html import fromstring
 from lxml.html import parse as html_parse
 
 from .errors import ReaderError
+from .xml import etree_to_dict
+
+try:
+    import oaipmh_scythe
+except ImportError:
+    oaipmh_scythe = None
 
 
 class BaseReader(ABC):
@@ -79,7 +87,12 @@ class TarReader(BaseReader):
     def read(self, item=None, *args, **kwargs):
         """Opens a tar archive or uses the given file pointer."""
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, tarfile.TarFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a TarFile (e.g. if it is a BytesIO), try to create a TarFile from the item.
+                with tarfile.open(mode=self._mode, fileobj=item) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with tarfile.open(self._origin, self._mode) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)
@@ -135,7 +148,12 @@ class ZipReader(BaseReader):
         """Opens a Zip archive or uses the given file pointer."""
         # https://docs.python.org/3/library/zipfile.html
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, zipfile.ZipFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a ZipFile (e.g. if it is a BytesIO), try to create a ZipFile from the item.
+                with zipfile.ZipFile(item, **self._options) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with zipfile.ZipFile(self._origin, **self._options) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)
@@ -206,34 +224,125 @@ class CSVReader(BaseReader):
 class XMLReader(BaseReader):
     """XML reader."""
 
-    @classmethod
-    def _etree_to_dict(cls, tree):
-        d = {tree.tag: {} if tree.attrib else None}
-        children = list(tree)
-        if children:
-            dd = defaultdict(list)
-            for dc in map(cls._etree_to_dict, children):
-                for k, v in dc.items():
-                    dd[k].append(v)
-            d = {tree.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
-        if tree.attrib:
-            d[tree.tag].update(("@" + k, v) for k, v in tree.attrib.items())
-        if tree.text:
-            text = tree.text.strip()
-            if children or tree.attrib:
-                if text:
-                    d[tree.tag]["#text"] = text
-            else:
-                d[tree.tag] = text
-        return d
+    def __init__(self, root_element=None, *args, **kwargs):
+        """Constructor."""
+        self.root_element = root_element
+        super().__init__(*args, **kwargs)
 
     def _iter(self, fp, *args, **kwargs):
         """Read and parse an XML file to dict."""
         # NOTE: We parse HTML, to skip XML validation and strip XML namespaces
-        xml_tree = html_parse(fp).getroot()
-        record = self._etree_to_dict(xml_tree)["html"]["body"].get("record")
-
-        if not record:
-            raise ReaderError(f"Record not found in XML entry.")
+        record = None
+        try:
+            xml_tree = fromstring(fp)
+            xml_dict = etree_to_dict(xml_tree)
+        except Exception as e:
+            xml_tree = html_parse(fp).getroot()
+            xml_dict = etree_to_dict(xml_tree)["html"]["body"]
+
+        if self.root_element:
+            record = xml_dict.get(self.root_element)
+            if not record:
+                raise ReaderError(
+                    f"Root element '{self.root_element}' not found in XML entry."
+                )
+        else:
+            record = xml_dict
 
         yield record
+
+
+class OAIPMHReader(BaseReader):
+    """OAIPMH reader."""
+
+    def __init__(
+        self,
+        *args,
+        base_url=None,
+        metadata_prefix=None,
+        set=None,
+        from_date=None,
+        until_date=None,
+        verb=None,
+        **kwargs,
+    ):
+        """Constructor."""
+        self._base_url = base_url
+        self._metadata_prefix = metadata_prefix if not None else "oai_dc"
+        self._set = set
+        self._until = until_date
+        self._from = from_date
+        self._verb = verb if not None else "ListRecords"
+        super().__init__(*args, **kwargs)
+
+    def _iter(self, scythe, *args, **kwargs):
+        """Read and parse an OAIPMH stream to dict."""
+
+        class OAIRecord(oaipmh_scythe.models.Record):
+            """An XML unpacking implementation for more complicated formats."""
+
+            def get_metadata(self):
+                """Extract and return the record's metadata as a dictionary."""
+                return xml_to_dict(
+                    self.xml.find(f".//{self._oai_namespace}metadata").getchildren()[0],
+                )
+
+        if self._verb == "ListRecords":
+            scythe.class_mapping["ListRecords"] = OAIRecord
+            try:
+                records = scythe.list_records(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for record in records:
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+        else:
+            scythe.class_mapping["GetRecord"] = OAIRecord
+            try:
+                headers = scythe.list_identifiers(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for header in headers:
+                    record = scythe.get_record(
+                        identifier=header.identifier,
+                        metadata_prefix=self._metadata_prefix,
+                    )
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads from item or opens the file descriptor from origin."""
+        if item:
+            raise NotImplementedError(
+                "OAIPMHReader does not support being chained after another reader"
+            )
+        else:
+            with oaipmh_scythe.Scythe(self._base_url) as scythe:
+                yield from self._iter(scythe=scythe, *args, **kwargs)
+
+
+def xml_to_dict(tree: etree._Element):
+    """Convert an XML tree to a dictionary.
+
+    This function takes an XML element tree and converts it into a dictionary.
+
+    Args:
+        tree: The root element of the XML tree to be converted.
+
+    Returns:
+        A dictionary with the key "record".
+    """
+    dict_obj = dict()
+    dict_obj["record"] = etree.tostring(tree)
+
+    return dict_obj

invenio_vocabularies/datastreams/tasks.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Data Streams Celery tasks."""
+
+from celery import shared_task
+
+from ..datastreams import StreamEntry
+from ..datastreams.factories import WriterFactory
+
+
+@shared_task(ignore_result=True)
+def write_entry(writer_config, entry):
+    """Write an entry.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: dictionary, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer_config)
+    writer.write(StreamEntry(entry))
+
+
+@shared_task(ignore_result=True)
+def write_many_entry(writer_config, entries):
+    """Write many entries.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: lisf ot dictionaries, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer_config)
+    stream_entries = [StreamEntry(entry) for entry in entries]
+    writer.write_many(stream_entries)

invenio_vocabularies/datastreams/transformers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,11 +9,11 @@
 """Transformers module."""
 
 from abc import ABC, abstractmethod
-from collections import defaultdict
 
 from lxml import etree
 
 from .errors import TransformerError
+from .xml import etree_to_dict
 
 
 class BaseTransformer(ABC):
@@ -32,42 +32,32 @@ class BaseTransformer(ABC):
 class XMLTransformer(BaseTransformer):
     """XML transformer."""
 
+    def __init__(self, root_element=None, *args, **kwargs):
+        """Initializes the transformer."""
+        self.root_element = root_element
+        super().__init__(*args, **kwargs)
+
     @classmethod
     def _xml_to_etree(cls, xml):
         """Converts XML to a lxml etree."""
         return etree.HTML(xml)
 
-    @classmethod
-    def _etree_to_dict(cls, tree):
-        d = {tree.tag: {} if tree.attrib else None}
-        children = list(tree)
-        if children:
-            dd = defaultdict(list)
-            for dc in map(cls._etree_to_dict, children):
-                for k, v in dc.items():
-                    dd[k].append(v)
-            d = {tree.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
-        if tree.attrib:
-            d[tree.tag].update(("@" + k, v) for k, v in tree.attrib.items())
-        if tree.text:
-            text = tree.text.strip()
-            if children or tree.attrib:
-                if text:
-                    d[tree.tag]["#text"] = text
-            else:
-                d[tree.tag] = text
-        return d
-
     def apply(self, stream_entry, **kwargs):
         """Applies the transformation to the stream entry.
 
         Requires the root element to be named "record".
         """
         xml_tree = self._xml_to_etree(stream_entry.entry)
-        record = self._etree_to_dict(xml_tree)["html"]["body"].get("record")
-
-        if not record:
-            raise TransformerError(f"Record not found in XML entry.")
+        xml_dict = etree_to_dict(xml_tree)["html"]["body"]
+
+        if self.root_element:
+            record = xml_dict.get(self.root_element)
+            if not record:
+                raise TransformerError(
+                    f"Root element '{self.root_element}' not found in XML entry."
+                )
+        else:
+            record = xml_dict
 
         stream_entry.entry = record
         return stream_entry