invenio-vocabularies 8.3.0__tar.gz → 9.1.0__tar.gz

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/CHANGES.rst

@@ -1,5 +1,5 @@
 ..
-    Copyright (C) 2020-2024 CERN.
+    Copyright (C) 2020-2025 CERN.
     Copyright (C) 2024-2025 Graz University of Technology.
 
     Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -9,6 +9,14 @@
 Changes
 =======
 
+Version v9.1.0 (released 2025-09-05)
+
+- installation: bump major version of invenio-jobs
+
+Version v9.0.0 (released 2025-09-05)
+
+- setup: bump major version of invenio-jobs
+
 Version v8.3.0 (released 2025-07-31)
 
 - subjects: Add search scheme field configuration to enable composite and filter query support in subject search functionality

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-vocabularies
-Version: 8.3.0
+Version: 9.1.0
 Summary: Invenio module for managing vocabularies.
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -15,7 +15,7 @@ License-File: AUTHORS.rst
 Requires-Dist: invenio-i18n<4.0.0,>=3.0.0
 Requires-Dist: invenio-records-resources<9.0.0,>=8.0.0
 Requires-Dist: invenio-administration<5.0.0,>=4.0.0
-Requires-Dist: invenio-jobs<5.0.0,>=4.0.0
+Requires-Dist: invenio-jobs<7.0.0,>=6.0.0
 Requires-Dist: lxml>=4.5.0
 Requires-Dist: pycountry>=24.0.0
 Requires-Dist: PyYAML>=5.4.1
@@ -79,7 +79,7 @@ Invenio module for managing vocabularies, based on Invenio-Records and Invenio-R
 - Helpers for importing vocabularies
 
 ..
-    Copyright (C) 2020-2024 CERN.
+    Copyright (C) 2020-2025 CERN.
     Copyright (C) 2024-2025 Graz University of Technology.
 
     Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -89,6 +89,14 @@ Invenio module for managing vocabularies, based on Invenio-Records and Invenio-R
 Changes
 =======
 
+Version v9.1.0 (released 2025-09-05)
+
+- installation: bump major version of invenio-jobs
+
+Version v9.0.0 (released 2025-09-05)
+
+- setup: bump major version of invenio-jobs
+
 Version v8.3.0 (released 2025-07-31)
 
 - subjects: Add search scheme field configuration to enable composite and filter query support in subject search functionality

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/__init__.py

@@ -11,6 +11,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "8.3.0"
+__version__ = "9.1.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/contrib/names/datastreams.py

@@ -69,12 +69,11 @@ class OrcidDataSyncReader(BaseReader):
         date_format_no_millis = "%Y-%m-%d %H:%M:%S"
 
         if self.since:
-            last_sync = datetime.now()
+            last_sync = datetime.strptime(self.since, date_format)
         else:
             last_sync = datetime.now() - timedelta(
                 **current_app.config["VOCABULARIES_ORCID_SYNC_SINCE"]
             )
-
         try:
             content = io.TextIOWrapper(fileobj, encoding="utf-8")
             csv_reader = csv.DictReader(content)
@@ -95,7 +94,10 @@ class OrcidDataSyncReader(BaseReader):
 
                 if last_modified_date < last_sync:
                     current_app.logger.debug(
-                        f"Skipping ORCiD {orcid} (last modified: {last_modified_date})"
+                        f"Skipping ORCiD {orcid}: last modified {last_modified_date} is older than cutoff {last_sync}"
+                    )
+                    current_app.logger.info(
+                        "Reached cutoff date. No more recent records to process."
                     )
                     break
                 current_app.logger.debug(f"Yielding ORCiD {orcid} for sync.")
@@ -285,7 +287,12 @@ class OrcidTransformer(BaseTransformer):
                     f"Family name not found in ORCiD entry for ORCiD ID: {orcid_id}."
                 )
             )
-        if not self._is_valid_name(given_names + family_name):
+        full_name = " ".join(
+            p.strip()
+            for p in (given_names, family_name)
+            if isinstance(p, str) and p.strip()
+        )
+        if not self._is_valid_name(full_name):
             errors.append(
                 TransformerError(
                     f"Invalid characters in name for ORCiD ID: {orcid_id}."

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/contrib/names/schema.py

@@ -80,18 +80,20 @@ class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
 
     @validates_schema
     def validate_affiliations(self, data, **kwargs):
-        """Validate names."""
+        """Validate and return distinct affiliations."""
         affiliations = data.get("affiliations", [])
         seen_names = set()
         for affiliation in affiliations:
             name = affiliation.get("name")
             if not affiliation.get("id") and name:
                 if name in seen_names:
-                    messages = [_("Duplicated affiliations.")]
-                    raise ValidationError({"affiliations": messages})
+                    affiliations.remove(affiliation)
                 else:
                     seen_names.add(name)
 
+        if affiliations:
+            data["affiliations"] = affiliations
+
     @post_load
     def update_name(self, data, **kwargs):
         """Update names for person.

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/datastreams/datastreams.py

@@ -9,6 +9,11 @@
 """Base data stream."""
 
 from flask import current_app
+from invenio_access.permissions import system_identity, system_user_id
+from invenio_access.utils import get_identity
+from invenio_accounts.proxies import current_datastore
+from invenio_jobs.logging.jobs import EMPTY_JOB_CTX, job_context
+from invenio_jobs.proxies import current_runs_service
 
 from .errors import ReaderError, TransformerError, WriterError
 
@@ -78,6 +83,14 @@ class DataStream:
     def process_batch(self, batch):
         """Process a batch of entries."""
         current_app.logger.info(f"Processing batch of size: {len(batch)}")
+        if job_context.get() is not EMPTY_JOB_CTX:
+            run_id = job_context.get()["run_id"]
+            current_runs_service.add_total_entries(
+                system_identity,
+                run_id=run_id,
+                job_id=job_context.get()["job_id"],
+                total_entries=len(batch),
+            )
         transformed_entries = []
         transformed_entries_with_errors = []
         for stream_entry in batch:
@@ -168,12 +181,36 @@ class DataStream:
 
         return stream_entry
 
+    def _prepare_async_context(self):
+        """Prepare the async context for writers."""
+        job_ctx = job_context.get()
+        run_id = job_ctx.get("run_id")
+        identity_id = job_ctx.get("identity_id")
+        job_id = job_ctx.get("job_id")
+        # System user needs to be handled separately because it doesn't exist in the database
+        if identity_id == system_user_id:
+            identity = system_identity
+        else:
+            user = current_datastore.get_user(identity_id)
+            if user is None:
+                raise ValueError(f"User with ID:{identity_id} not found.")
+            identity = get_identity(user)
+
+        subtask_run = current_runs_service.create_subtask_run(
+            identity, parent_run_id=run_id, job_id=job_id
+        )
+        return str(subtask_run.id)
+
     def write(self, stream_entry, *args, **kwargs):
-        """Apply the transformations to an stream_entry."""
+        """Write a single stream entry."""
         current_app.logger.debug(f"Writing entry: {stream_entry.entry}")
         for writer in self._writers:
             try:
-                writer.write(stream_entry)
+                if writer.is_async and job_context.get() is not EMPTY_JOB_CTX:
+                    subtask_run_id = self._prepare_async_context()
+                    writer.write(stream_entry, subtask_run_id=subtask_run_id)
+                else:
+                    writer.write(stream_entry)
             except WriterError as err:
                 current_app.logger.error(f"Writer error: {str(err)}")
                 stream_entry.errors.append(f"{writer.__class__.__name__}: {str(err)}")
@@ -181,10 +218,21 @@ class DataStream:
         return stream_entry
 
     def batch_write(self, stream_entries, *args, **kwargs):
-        """Apply the transformations to an stream_entry. Errors are handler in the service layer."""
+        """Write a batch of stream entries."""
         current_app.logger.debug(f"Batch writing entries: {len(stream_entries)}")
         for writer in self._writers:
-            yield from writer.write_many(stream_entries)
+            try:
+                if writer.is_async and job_context.get() is not EMPTY_JOB_CTX:
+                    subtask_run_id = self._prepare_async_context()
+                    yield from writer.write_many(
+                        stream_entries, subtask_run_id=subtask_run_id
+                    )
+                else:
+                    yield from writer.write_many(stream_entries)
+            except WriterError as err:
+                current_app.logger.error(f"Writer error: {str(err)}")
+                for entry in stream_entries:
+                    entry.errors.append(f"{writer.__class__.__name__}: {str(err)}")
 
     def total(self, *args, **kwargs):
         """The total of entries obtained from the origin."""

invenio_vocabularies-9.1.0/invenio_vocabularies/datastreams/tasks.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Data Streams Celery tasks."""
+
+from celery import shared_task
+from flask import current_app
+from invenio_access.permissions import system_identity
+from invenio_jobs.logging.jobs import EMPTY_JOB_CTX, job_context
+from invenio_jobs.proxies import current_runs_service
+
+from ..datastreams import StreamEntry
+from ..datastreams.factories import WriterFactory
+
+
+@shared_task(ignore_result=True)
+def write_entry(writer_config, entry, subtask_run_id=None):
+    """Write an entry.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: dictionary, StreamEntry is not serializable.
+    """
+    job_ctx = job_context.get()
+    job_id = job_ctx.get("job_id", None) if job_ctx is not EMPTY_JOB_CTX else None
+    if subtask_run_id and job_id:
+        subtask_run = current_runs_service.get(
+            system_identity, job_id=job_id, run_id=subtask_run_id
+        )
+        current_runs_service.start_processing_subtask(
+            system_identity, subtask_run.id, job_id=job_id
+        )
+
+    writer = WriterFactory.create(config=writer_config)
+    try:
+        processed_stream_entry = writer.write(StreamEntry(entry))
+        errored_entries_count = 1 if processed_stream_entry.errors else 0
+        inserted_count = 1 if processed_stream_entry.op_type == "create" else 0
+        updated_count = 1 if processed_stream_entry.op_type == "update" else 0
+        if subtask_run_id and job_id:
+            current_runs_service.finalize_subtask(
+                system_identity,
+                subtask_run_id,
+                job_id,
+                success=True if not processed_stream_entry.errors else False,
+                errored_entries_count=errored_entries_count,
+                inserted_entries_count=inserted_count,
+                updated_entries_count=updated_count,
+            )
+    except Exception as exc:
+        current_app.logger.error(f"Error writing entry {entry}: {exc}")
+        if subtask_run_id and job_id:
+            current_runs_service.finalize_subtask(
+                system_identity,
+                subtask_run_id,
+                job_id,
+                success=False,
+                errored_entries_count=1,
+            )
+
+
+@shared_task(ignore_result=True)
+def write_many_entry(writer_config, entries, subtask_run_id=None):
+    """Write many entries.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: lisf ot dictionaries, StreamEntry is not serializable.
+    """
+    job_ctx = job_context.get()
+    job_id = job_ctx.get("job_id", None) if job_ctx is not EMPTY_JOB_CTX else None
+    if subtask_run_id and job_id:
+        subtask_run = current_runs_service.get(
+            system_identity, job_id=job_id, run_id=subtask_run_id
+        )
+        current_runs_service.start_processing_subtask(
+            system_identity, subtask_run.id, job_id=job_id
+        )
+    writer = WriterFactory.create(config=writer_config)
+    stream_entries = [StreamEntry(entry) for entry in entries]
+    try:
+        processed_stream_entries = writer.write_many(stream_entries)
+        errored_entries_count = sum(
+            1 for entry in processed_stream_entries if entry.errors
+        )
+        inserted_count = sum(
+            1 for entry in processed_stream_entries if entry.op_type == "create"
+        )
+        updated_count = sum(
+            1 for entry in processed_stream_entries if entry.op_type == "update"
+        )
+        if subtask_run_id and job_id:
+            current_runs_service.finalize_subtask(
+                system_identity,
+                subtask_run_id,
+                job_id,
+                success=True,
+                errored_entries_count=errored_entries_count,
+                inserted_entries_count=inserted_count,
+                updated_entries_count=updated_count,
+            )
+    except Exception as exc:
+        current_app.logger.error(
+            f"Error writing entries {entries}: {exc}. The errorred entries count might be incorrect as an entire batch might have failed"
+        )
+        if subtask_run_id and job_id:
+            current_runs_service.finalize_subtask(
+                system_identity,
+                subtask_run_id,
+                job_id,
+                success=False,
+            )

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/datastreams/writers.py

@@ -28,6 +28,8 @@ from .tasks import write_entry, write_many_entry
 class BaseWriter(ABC):
     """Base writer."""
 
+    is_async = False
+
     def __init__(self, *args, **kwargs):
         """Base initialization logic."""
         # Add any base initialization here if needed
@@ -91,7 +93,9 @@ class ServiceWriter(BaseWriter):
         current = self._resolve(vocab_id)
         updated = dict(current.to_dict(), **entry)
         current_app.logger.debug(f"Updating entry with ID: {vocab_id}")
-        return StreamEntry(self._service.update(self._identity, vocab_id, updated))
+        return StreamEntry(
+            self._service.update(self._identity, vocab_id, updated), op_type="update"
+        )
 
     def write(self, stream_entry, *args, **kwargs):
         """Writes the input entry using a given service."""
@@ -102,7 +106,9 @@ class ServiceWriter(BaseWriter):
             if self._insert:
                 try:
                     current_app.logger.debug("Inserting entry.")
-                    return StreamEntry(self._service.create(self._identity, entry))
+                    return StreamEntry(
+                        self._service.create(self._identity, entry), op_type="create"
+                    )
                 except PIDAlreadyExists:
                     if not self._update:
                         raise WriterError([f"Vocabulary entry already exists: {entry}"])
@@ -182,6 +188,8 @@ class YamlWriter(BaseWriter):
 class AsyncWriter(BaseWriter):
     """Writes the entries asynchronously (celery task)."""
 
+    is_async = True
+
     def __init__(self, writer, *args, **kwargs):
         """Constructor.
 
@@ -190,16 +198,25 @@ class AsyncWriter(BaseWriter):
         super().__init__(*args, **kwargs)
         self._writer = writer
 
-    def write(self, stream_entry, *args, **kwargs):
-        """Launches a celery task to write an entry."""
-        write_entry.delay(self._writer, stream_entry.entry)
+    def write(self, stream_entry, subtask_run_id=None, *args, **kwargs):
+        """Launches a celery task to write an entry with a delay."""
+        # Add some delay to avoid processing the tasks too fast
+        write_entry.apply_async(
+            args=(self._writer, stream_entry.entry, subtask_run_id), countdown=1
+        )
 
         return stream_entry
 
-    def write_many(self, stream_entries, *args, **kwargs):
-        """Launches a celery task to write an entry."""
-        write_many_entry.delay(
-            self._writer, [stream_entry.entry for stream_entry in stream_entries]
+    def write_many(self, stream_entries, subtask_run_id=None, *args, **kwargs):
+        """Launches a celery task to write entries with a delay."""
+        # Add some delay to avoid processing the tasks too fast
+        write_many_entry.apply_async(
+            args=(
+                self._writer,
+                [stream_entry.entry for stream_entry in stream_entries],
+                subtask_run_id,
+            ),
+            countdown=1,
         )
 
         return stream_entries

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/jobs.py

@@ -8,6 +8,9 @@
 
 """Jobs module."""
 
+from datetime import datetime, timedelta
+
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 from invenio_jobs.jobs import JobType
 
@@ -177,3 +180,28 @@ class ImportORCIDJob(ProcessDataStreamJob):
             # It is the responsibility of the reader to handle it or ignore it
             reader["args"] = {**reader.get("args", {}), "since": str(since)}
         return task_args
+
+    @classmethod
+    def _build_task_arguments(cls, job_obj, since=None, custom_args=None, **kwargs):
+        """Build dict of arguments injected on task execution.
+
+        :param job_obj (Job): the Job object.
+        :param since (datetime): last time the job was executed.
+        :param custom_args (dict): when provided, takes precedence over any other
+            provided argument.
+        :return: a dict of arguments to be injected on task execution.
+        """
+        if custom_args:
+            return custom_args
+
+        if since is None:
+            """We set since to a time in the past defined by the VOCABULARIES_ORCID_SYNC_SINCE."""
+
+            since = datetime.now() - timedelta(
+                **current_app.config["VOCABULARIES_ORCID_SYNC_SINCE"]
+            )
+        """
+        Otherwise, since is already specified as a datetime with a timezone (see PredefinedArgsSchema) or we have never
+        run the job before so there is no logical value.
+        """
+        return {**cls.build_task_arguments(job_obj, since=since, **kwargs)}

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies/services/tasks.py

@@ -30,7 +30,9 @@ def process_datastream(config):
             for err in result.errors:
                 current_app.logger.error(err)
             entries_with_errors += 1
+
     if entries_with_errors:
         raise TaskExecutionPartialError(
-            message=f"Task execution succeeded with {entries_with_errors} entries with errors."
+            message=f"Task execution succeeded with {entries_with_errors} entries with errors.",
+            errored_entries_count=entries_with_errors,
         )

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-vocabularies
-Version: 8.3.0
+Version: 9.1.0
 Summary: Invenio module for managing vocabularies.
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -15,7 +15,7 @@ License-File: AUTHORS.rst
 Requires-Dist: invenio-i18n<4.0.0,>=3.0.0
 Requires-Dist: invenio-records-resources<9.0.0,>=8.0.0
 Requires-Dist: invenio-administration<5.0.0,>=4.0.0
-Requires-Dist: invenio-jobs<5.0.0,>=4.0.0
+Requires-Dist: invenio-jobs<7.0.0,>=6.0.0
 Requires-Dist: lxml>=4.5.0
 Requires-Dist: pycountry>=24.0.0
 Requires-Dist: PyYAML>=5.4.1
@@ -79,7 +79,7 @@ Invenio module for managing vocabularies, based on Invenio-Records and Invenio-R
 - Helpers for importing vocabularies
 
 ..
-    Copyright (C) 2020-2024 CERN.
+    Copyright (C) 2020-2025 CERN.
     Copyright (C) 2024-2025 Graz University of Technology.
 
     Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -89,6 +89,14 @@ Invenio module for managing vocabularies, based on Invenio-Records and Invenio-R
 Changes
 =======
 
+Version v9.1.0 (released 2025-09-05)
+
+- installation: bump major version of invenio-jobs
+
+Version v9.0.0 (released 2025-09-05)
+
+- setup: bump major version of invenio-jobs
+
 Version v8.3.0 (released 2025-07-31)
 
 - subjects: Add search scheme field configuration to enable composite and filter query support in subject search functionality

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/invenio_vocabularies.egg-info/requires.txt

@@ -1,7 +1,7 @@
 invenio-i18n<4.0.0,>=3.0.0
 invenio-records-resources<9.0.0,>=8.0.0
 invenio-administration<5.0.0,>=4.0.0
-invenio-jobs<5.0.0,>=4.0.0
+invenio-jobs<7.0.0,>=6.0.0
 lxml>=4.5.0
 pycountry>=24.0.0
 PyYAML>=5.4.1

{invenio_vocabularies-8.3.0 → invenio_vocabularies-9.1.0}/setup.cfg

@@ -21,7 +21,7 @@ install_requires =
 	invenio-i18n>=3.0.0,<4.0.0
 	invenio-records-resources>=8.0.0,<9.0.0
 	invenio-administration>=4.0.0,<5.0.0
-	invenio-jobs>=4.0.0,<5.0.0
+	invenio-jobs>=6.0.0,<7.0.0
 	lxml>=4.5.0
 	pycountry>=24.0.0
 	PyYAML>=5.4.1

invenio_vocabularies-8.3.0/invenio_vocabularies/datastreams/tasks.py

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2022-2024 CERN.
-#
-# Invenio-Vocabularies is free software; you can redistribute it and/or
-# modify it under the terms of the MIT License; see LICENSE file for more
-# details.
-
-"""Data Streams Celery tasks."""
-
-from celery import shared_task
-
-from ..datastreams import StreamEntry
-from ..datastreams.factories import WriterFactory
-
-
-@shared_task(ignore_result=True)
-def write_entry(writer_config, entry):
-    """Write an entry.
-
-    :param writer: writer configuration as accepted by the WriterFactory.
-    :param entry: dictionary, StreamEntry is not serializable.
-    """
-    writer = WriterFactory.create(config=writer_config)
-    writer.write(StreamEntry(entry))
-
-
-@shared_task(ignore_result=True)
-def write_many_entry(writer_config, entries):
-    """Write many entries.
-
-    :param writer: writer configuration as accepted by the WriterFactory.
-    :param entry: lisf ot dictionaries, StreamEntry is not serializable.
-    """
-    writer = WriterFactory.create(config=writer_config)
-    stream_entries = [StreamEntry(entry) for entry in entries]
-    writer.write_many(stream_entries)