invenio-vocabularies 2.3.1__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/subjects/subjects.py

@@ -9,9 +9,11 @@
 
 """Vocabulary subjects."""
 
+from flask_resources import JSONSerializer, ResponseHandler
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.resources.records.headers import etag_headers
 
 from ...records.pidprovider import PIDProviderFactory
 from ...records.systemfields import BaseVocabularyPIDFieldContext
@@ -42,4 +44,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/subjects",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

invenio_vocabularies/datastreams/datastreams.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,17 +8,41 @@
 
 """Base data stream."""
 
+from flask import current_app
+
 from .errors import ReaderError, TransformerError, WriterError
 
 
 class StreamEntry:
     """Object to encapsulate streams processing."""
 
-    def __init__(self, entry, errors=None):
-        """Constructor."""
+    def __init__(self, entry, record=None, errors=None, op_type=None, exc=None):
+        """Constructor for the StreamEntry class.
+
+        :param entry (object): The entry object, usually a record dict.
+        :param record (object): The record object, usually a record class.
+        :param errors (list, optional): List of errors. Defaults to None.
+        :param op_type (str, optional): The operation type. Defaults to None.
+        :param exc (str, optional): The raised unhandled exception. Defaults to None.
+        """
         self.entry = entry
+        self.record = record
         self.filtered = False
         self.errors = errors or []
+        self.op_type = op_type
+        self.exc = exc
+
+    def log_errors(self, logger=None):
+        """Log the errors using the provided logger or the default logger.
+
+        :param logger (logging.Logger, optional): Logger instance to use. Defaults to None.
+        """
+        if logger is None:
+            logger = current_app.logger
+        for error in self.errors:
+            logger.error(f"Error in entry {self.entry}: {error}")
+        if self.exc:
+            logger.error(f"Exception in entry {self.entry}: {self.exc}")
 
 
 class DataStream:
@@ -39,15 +63,10 @@ class DataStream:
         """Checks if an stream_entry should be filtered out (skipped)."""
         return False
 
-    def process(self, *args, **kwargs):
-        """Iterates over the entries.
-
-        Uses the reader to get the raw entries and transforms them.
-        It will iterate over the `StreamEntry` objects returned by
-        the reader, apply the transformations and yield the result of
-        writing it.
-        """
-        for stream_entry in self.read():
+    def process_batch(self, batch, write_many=False):
+        """Process a batch of entries."""
+        transformed_entries = []
+        for stream_entry in batch:
             if stream_entry.errors:
                 yield stream_entry  # reading errors
             else:
@@ -58,7 +77,31 @@ class DataStream:
                     transformed_entry.filtered = True
                     yield transformed_entry
                 else:
-                    yield self.write(transformed_entry)
+                    transformed_entries.append(transformed_entry)
+        if transformed_entries:
+            if write_many:
+                yield from self.batch_write(transformed_entries)
+            else:
+                yield from (self.write(entry) for entry in transformed_entries)
+
+    def process(self, batch_size=100, write_many=False, *args, **kwargs):
+        """Iterates over the entries.
+
+        Uses the reader to get the raw entries and transforms them.
+        It will iterate over the `StreamEntry` objects returned by
+        the reader, apply the transformations and yield the result of
+        writing it.
+        """
+        batch = []
+        for stream_entry in self.read():
+            batch.append(stream_entry)
+            if len(batch) >= batch_size:
+                yield from self.process_batch(batch, write_many=write_many)
+                batch = []
+
+        # Process any remaining entries in the last batch
+        if batch:
+            yield from self.process_batch(batch, write_many=write_many)
 
     def read(self):
         """Recursively read the entries."""
@@ -107,6 +150,11 @@ class DataStream:
 
         return stream_entry
 
+    def batch_write(self, stream_entries, *args, **kwargs):
+        """Apply the transformations to an stream_entry. Errors are handler in the service layer."""
+        for writer in self._writers:
+            yield from writer.write_many(stream_entries)
+
     def total(self, *args, **kwargs):
         """The total of entries obtained from the origin."""
         raise NotImplementedError()

invenio_vocabularies/datastreams/factories.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -36,7 +36,6 @@ class Factory:
         try:
             type_ = config["type"]
             args = config.get("args", {})
-
             return cls.options()[type_](**args)
         except KeyError:
             raise FactoryError(name=cls.FACTORY_NAME, key=type_)

invenio_vocabularies/datastreams/readers.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C)      2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -15,14 +16,21 @@ import re
 import tarfile
 import zipfile
 from abc import ABC, abstractmethod
-from collections import defaultdict
 from json.decoder import JSONDecodeError
 
 import requests
 import yaml
+from lxml import etree
+from lxml.html import fromstring
 from lxml.html import parse as html_parse
 
 from .errors import ReaderError
+from .xml import etree_to_dict
+
+try:
+    import oaipmh_scythe
+except ImportError:
+    oaipmh_scythe = None
 
 
 class BaseReader(ABC):
@@ -79,7 +87,12 @@ class TarReader(BaseReader):
     def read(self, item=None, *args, **kwargs):
         """Opens a tar archive or uses the given file pointer."""
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, tarfile.TarFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a TarFile (e.g. if it is a BytesIO), try to create a TarFile from the item.
+                with tarfile.open(mode=self._mode, fileobj=item) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with tarfile.open(self._origin, self._mode) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)
@@ -135,7 +148,12 @@ class ZipReader(BaseReader):
         """Opens a Zip archive or uses the given file pointer."""
         # https://docs.python.org/3/library/zipfile.html
         if item:
-            yield from self._iter(fp=item, *args, **kwargs)
+            if isinstance(item, zipfile.ZipFile):
+                yield from self._iter(fp=item, *args, **kwargs)
+            else:
+                # If the item is not already a ZipFile (e.g. if it is a BytesIO), try to create a ZipFile from the item.
+                with zipfile.ZipFile(item, **self._options) as archive:
+                    yield from self._iter(fp=archive, *args, **kwargs)
         else:
             with zipfile.ZipFile(self._origin, **self._options) as archive:
                 yield from self._iter(fp=archive, *args, **kwargs)
@@ -206,34 +224,125 @@ class CSVReader(BaseReader):
 class XMLReader(BaseReader):
     """XML reader."""
 
-    @classmethod
-    def _etree_to_dict(cls, tree):
-        d = {tree.tag: {} if tree.attrib else None}
-        children = list(tree)
-        if children:
-            dd = defaultdict(list)
-            for dc in map(cls._etree_to_dict, children):
-                for k, v in dc.items():
-                    dd[k].append(v)
-            d = {tree.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
-        if tree.attrib:
-            d[tree.tag].update(("@" + k, v) for k, v in tree.attrib.items())
-        if tree.text:
-            text = tree.text.strip()
-            if children or tree.attrib:
-                if text:
-                    d[tree.tag]["#text"] = text
-            else:
-                d[tree.tag] = text
-        return d
+    def __init__(self, root_element=None, *args, **kwargs):
+        """Constructor."""
+        self.root_element = root_element
+        super().__init__(*args, **kwargs)
 
     def _iter(self, fp, *args, **kwargs):
         """Read and parse an XML file to dict."""
         # NOTE: We parse HTML, to skip XML validation and strip XML namespaces
-        xml_tree = html_parse(fp).getroot()
-        record = self._etree_to_dict(xml_tree)["html"]["body"].get("record")
-
-        if not record:
-            raise ReaderError(f"Record not found in XML entry.")
+        record = None
+        try:
+            xml_tree = fromstring(fp)
+            xml_dict = etree_to_dict(xml_tree)
+        except Exception as e:
+            xml_tree = html_parse(fp).getroot()
+            xml_dict = etree_to_dict(xml_tree)["html"]["body"]
+
+        if self.root_element:
+            record = xml_dict.get(self.root_element)
+            if not record:
+                raise ReaderError(
+                    f"Root element '{self.root_element}' not found in XML entry."
+                )
+        else:
+            record = xml_dict
 
         yield record
+
+
+class OAIPMHReader(BaseReader):
+    """OAIPMH reader."""
+
+    def __init__(
+        self,
+        *args,
+        base_url=None,
+        metadata_prefix=None,
+        set=None,
+        from_date=None,
+        until_date=None,
+        verb=None,
+        **kwargs,
+    ):
+        """Constructor."""
+        self._base_url = base_url
+        self._metadata_prefix = metadata_prefix if not None else "oai_dc"
+        self._set = set
+        self._until = until_date
+        self._from = from_date
+        self._verb = verb if not None else "ListRecords"
+        super().__init__(*args, **kwargs)
+
+    def _iter(self, scythe, *args, **kwargs):
+        """Read and parse an OAIPMH stream to dict."""
+
+        class OAIRecord(oaipmh_scythe.models.Record):
+            """An XML unpacking implementation for more complicated formats."""
+
+            def get_metadata(self):
+                """Extract and return the record's metadata as a dictionary."""
+                return xml_to_dict(
+                    self.xml.find(f".//{self._oai_namespace}metadata").getchildren()[0],
+                )
+
+        if self._verb == "ListRecords":
+            scythe.class_mapping["ListRecords"] = OAIRecord
+            try:
+                records = scythe.list_records(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for record in records:
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+        else:
+            scythe.class_mapping["GetRecord"] = OAIRecord
+            try:
+                headers = scythe.list_identifiers(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for header in headers:
+                    record = scythe.get_record(
+                        identifier=header.identifier,
+                        metadata_prefix=self._metadata_prefix,
+                    )
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads from item or opens the file descriptor from origin."""
+        if item:
+            raise NotImplementedError(
+                "OAIPMHReader does not support being chained after another reader"
+            )
+        else:
+            with oaipmh_scythe.Scythe(self._base_url) as scythe:
+                yield from self._iter(scythe=scythe, *args, **kwargs)
+
+
+def xml_to_dict(tree: etree._Element):
+    """Convert an XML tree to a dictionary.
+
+    This function takes an XML element tree and converts it into a dictionary.
+
+    Args:
+        tree: The root element of the XML tree to be converted.
+
+    Returns:
+        A dictionary with the key "record".
+    """
+    dict_obj = dict()
+    dict_obj["record"] = etree.tostring(tree)
+
+    return dict_obj

invenio_vocabularies/datastreams/tasks.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Data Streams Celery tasks."""
+
+from celery import shared_task
+
+from ..datastreams import StreamEntry
+from ..datastreams.factories import WriterFactory
+
+
+@shared_task(ignore_result=True)
+def write_entry(writer_config, entry):
+    """Write an entry.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: dictionary, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer_config)
+    writer.write(StreamEntry(entry))
+
+
+@shared_task(ignore_result=True)
+def write_many_entry(writer_config, entries):
+    """Write many entries.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: lisf ot dictionaries, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer_config)
+    stream_entries = [StreamEntry(entry) for entry in entries]
+    writer.write_many(stream_entries)

invenio_vocabularies/datastreams/transformers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,11 +9,11 @@
 """Transformers module."""
 
 from abc import ABC, abstractmethod
-from collections import defaultdict
 
 from lxml import etree
 
 from .errors import TransformerError
+from .xml import etree_to_dict
 
 
 class BaseTransformer(ABC):
@@ -32,42 +32,32 @@ class BaseTransformer(ABC):
 class XMLTransformer(BaseTransformer):
     """XML transformer."""
 
+    def __init__(self, root_element=None, *args, **kwargs):
+        """Initializes the transformer."""
+        self.root_element = root_element
+        super().__init__(*args, **kwargs)
+
     @classmethod
     def _xml_to_etree(cls, xml):
         """Converts XML to a lxml etree."""
         return etree.HTML(xml)
 
-    @classmethod
-    def _etree_to_dict(cls, tree):
-        d = {tree.tag: {} if tree.attrib else None}
-        children = list(tree)
-        if children:
-            dd = defaultdict(list)
-            for dc in map(cls._etree_to_dict, children):
-                for k, v in dc.items():
-                    dd[k].append(v)
-            d = {tree.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
-        if tree.attrib:
-            d[tree.tag].update(("@" + k, v) for k, v in tree.attrib.items())
-        if tree.text:
-            text = tree.text.strip()
-            if children or tree.attrib:
-                if text:
-                    d[tree.tag]["#text"] = text
-            else:
-                d[tree.tag] = text
-        return d
-
     def apply(self, stream_entry, **kwargs):
         """Applies the transformation to the stream entry.
 
         Requires the root element to be named "record".
         """
         xml_tree = self._xml_to_etree(stream_entry.entry)
-        record = self._etree_to_dict(xml_tree)["html"]["body"].get("record")
-
-        if not record:
-            raise TransformerError(f"Record not found in XML entry.")
+        xml_dict = etree_to_dict(xml_tree)["html"]["body"]
+
+        if self.root_element:
+            record = xml_dict.get(self.root_element)
+            if not record:
+                raise TransformerError(
+                    f"Root element '{self.root_element}' not found in XML entry."
+                )
+        else:
+            record = xml_dict
 
         stream_entry.entry = record
         return stream_entry

invenio_vocabularies/datastreams/writers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -13,18 +13,25 @@ from pathlib import Path
 
 import yaml
 from invenio_access.permissions import system_identity
-from invenio_pidstore.errors import PIDAlreadyExists
+from invenio_pidstore.errors import PIDAlreadyExists, PIDDoesNotExistError
 from invenio_records.systemfields.relations.errors import InvalidRelationValue
 from invenio_records_resources.proxies import current_service_registry
 from marshmallow import ValidationError
+from sqlalchemy.exc import NoResultFound
 
 from .datastreams import StreamEntry
 from .errors import WriterError
+from .tasks import write_entry, write_many_entry
 
 
 class BaseWriter(ABC):
     """Base writer."""
 
+    def __init__(self, *args, **kwargs):
+        """Base initialization logic."""
+        # Add any base initialization here if needed
+        pass
+
     @abstractmethod
     def write(self, stream_entry, *args, **kwargs):
         """Writes the input stream entry to the target output.
@@ -35,16 +42,29 @@ class BaseWriter(ABC):
         """
         pass
 
+    @abstractmethod
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input streams entry to the target output.
+
+        :returns: A List of StreamEntry. The result of writing the entry.
+                  Raises WriterException in case of errors.
+
+        """
+        pass
+
 
 class ServiceWriter(BaseWriter):
     """Writes the entries to an RDM instance using a Service object."""
 
-    def __init__(self, service_or_name, *args, identity=None, update=False, **kwargs):
+    def __init__(
+        self, service_or_name, *args, identity=None, insert=True, update=False, **kwargs
+    ):
         """Constructor.
 
         :param service_or_name: a service instance or a key of the
                                 service registry.
         :param identity: access identity.
+        :param insert: if True it will insert records which do not exist.
         :param update: if True it will update records if they exist.
         """
         if isinstance(service_or_name, str):
@@ -52,6 +72,7 @@ class ServiceWriter(BaseWriter):
 
         self._service = service_or_name
         self._identity = identity or system_identity
+        self._insert = insert
         self._update = update
 
         super().__init__(*args, **kwargs)
@@ -63,20 +84,47 @@ class ServiceWriter(BaseWriter):
     def _resolve(self, id_):
         return self._service.read(self._identity, id_)
 
+    def _do_update(self, entry):
+        vocab_id = self._entry_id(entry)
+        current = self._resolve(vocab_id)
+        combined_dict = current.to_dict()
+
+        # Update fields from entry
+        for key, value in entry.items():
+            if key in combined_dict:
+                if isinstance(combined_dict[key], list) and isinstance(value, list):
+                    combined_dict[key].extend(
+                        item for item in value if item not in combined_dict[key]
+                    )
+                else:
+                    combined_dict[key] = value
+            else:
+                combined_dict[key] = value
+
+        return StreamEntry(
+            self._service.update(self._identity, vocab_id, combined_dict)
+        )
+
     def write(self, stream_entry, *args, **kwargs):
         """Writes the input entry using a given service."""
         entry = stream_entry.entry
+
         try:
-            try:
-                return StreamEntry(self._service.create(self._identity, entry))
-            except PIDAlreadyExists:
-                if not self._update:
-                    raise WriterError([f"Vocabulary entry already exists: {entry}"])
-                vocab_id = self._entry_id(entry)
-                current = self._resolve(vocab_id)
-                updated = dict(current.to_dict(), **entry)
-                return StreamEntry(
-                    self._service.update(self._identity, vocab_id, updated)
+            if self._insert:
+                try:
+                    return StreamEntry(self._service.create(self._identity, entry))
+                except PIDAlreadyExists:
+                    if not self._update:
+                        raise WriterError([f"Vocabulary entry already exists: {entry}"])
+                    return self._do_update(entry)
+            elif self._update:
+                try:
+                    return self._do_update(entry)
+                except (NoResultFound, PIDDoesNotExistError):
+                    raise WriterError([f"Vocabulary entry does not exist: {entry}"])
+            else:
+                raise WriterError(
+                    ["Writer wrongly configured to not insert and to not update"]
                 )
 
         except ValidationError as err:
@@ -85,6 +133,25 @@ class ServiceWriter(BaseWriter):
             # TODO: Check if we can get the error message easier
             raise WriterError([{"InvalidRelationValue": err.args[0]}])
 
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input entries using a given service."""
+        entries = [entry.entry for entry in stream_entries]
+        entries_with_id = [(self._entry_id(entry), entry) for entry in entries]
+        results = self._service.create_or_update_many(self._identity, entries_with_id)
+        stream_entries_processed = []
+        for entry, result in zip(entries, results):
+            processed_stream_entry = StreamEntry(
+                entry=entry,
+                record=result.record,
+                errors=result.errors,
+                op_type=result.op_type,
+                exc=result.exc,
+            )
+            processed_stream_entry.log_errors()
+            stream_entries_processed.append(processed_stream_entry)
+
+        return stream_entries_processed
+
 
 class YamlWriter(BaseWriter):
     """Writes the entries to a YAML file."""
@@ -103,6 +170,41 @@ class YamlWriter(BaseWriter):
         with open(self._filepath, "a") as file:
             # made into array for safer append
             # will always read array (good for reader)
-            yaml.safe_dump([stream_entry.entry], file)
+            yaml.safe_dump([stream_entry.entry], file, allow_unicode=True)
 
         return stream_entry
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the yaml input entries."""
+        with open(self._filepath, "a") as file:
+            yaml.safe_dump(
+                [stream_entry.entry for stream_entry in stream_entries],
+                file,
+                allow_unicode=True,
+            )
+
+
+class AsyncWriter(BaseWriter):
+    """Writes the entries asynchronously (celery task)."""
+
+    def __init__(self, writer, *args, **kwargs):
+        """Constructor.
+
+        :param writer: writer to use.
+        """
+        super().__init__(*args, **kwargs)
+        self._writer = writer
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_entry.delay(self._writer, stream_entry.entry)
+
+        return stream_entry
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_many_entry.delay(
+            self._writer, [stream_entry.entry for stream_entry in stream_entries]
+        )
+
+        return stream_entries