invenio-app-rdm 5.0.0.dev0__py2.py3-none-any.whl → 14.0.0b4.dev4__py2.py3-none-any.whl

invenio_app_rdm/upgrade_scripts/migrate_13_0_to_14_0.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 13.0 to 14.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 13.0 to 14.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested for the following scenarios:
+1. Draft with resource type publication-thesis
+2. Record with resource type publication-thesis with a DOI and no draft
+3. Record with resource type publication-thesis with a no DOI and an existing draft
+4. Records with multiple versions
+"""
+
+from click import secho
+from invenio_access.permissions import system_identity
+from invenio_db import db
+from invenio_drafts_resources.resources.records.errors import DraftNotCreatedError
+from invenio_rdm_records.proxies import current_rdm_records_service as records_service
+from invenio_search.api import RecordsSearchV2
+
+
+def run_upgrade(migrate_record, migrate_draft):
+    """Run upgrade on selected records and drafts.
+
+    Args:
+        migrate_record (callable): Function to migrate a record.
+        migrate_draft (callable): Function to migrate a draft.
+    """
+    errored_record_ids = []
+    errored_draft_ids = []
+
+    # Handle published records
+    published_records = (
+        RecordsSearchV2(index=records_service.record_cls.index._name)
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )  # Only need to fetch the record IDs to make the query faster
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    published_record_ids = [result["id"] for result in published_records]
+    for record_id in published_record_ids:
+        try:
+            migrate_record(record_id)
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            secho(f"Record {record_id} failed to update", fg="red")
+            errored_record_ids.append((record_id, error))
+
+    # Handle draft records
+    draft_records = (
+        RecordsSearchV2(index=records_service.draft_cls.index._name)
+        .filter("term", has_draft=False)
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    draft_record_ids = [result["id"] for result in draft_records]
+    for draft_id in draft_record_ids:
+        try:
+            migrate_draft(draft_id)
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            secho(f"Draft {draft_id} failed to update", fg="red")
+            errored_draft_ids.append((draft_id, error))
+
+    print(f"Errored record IDs:", *errored_record_ids, sep="\n")
+    print(f"Errored draft IDs:", *errored_draft_ids, sep="\n")
+
+
+def run_update_for_resource_type():
+    """Run update for resource type."""
+
+    def migrate_resource_type_in_record(record_id):
+        """
+        Update resource type from publication-thesis to publication-dissertation.
+
+        We go through the service layer to automatically trigger the DOI update and re-indexing.
+        """
+        secho(f"Updating resource type for record {record_id}", fg="yellow")
+        record = records_service.read(system_identity, record_id)
+        if record.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in record.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
+            secho(
+                f"Skipping record <{record.id}> because it doesn't have resource-type 'publication-thesis'!",
+                fg="yellow",
+            )
+            return
+
+        try:
+            draft = records_service.read_draft(system_identity, record.id)
+            # Step 1: Update the resource type in the record via low-level API
+            # We need to make sure we don't publish the record with different metadata
+            secho(
+                f"Record <{record.id}> has an existing draft <{draft.id}>! Updating record via low-level API.",
+                fg="yellow",
+            )
+            # Update the record directly without affecting the draft
+            if (
+                record._record["metadata"]["resource_type"]["id"]
+                == "publication-thesis"
+            ):
+                record._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in record._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            # Save the record changes and reindex
+            secho(
+                f"Record <{record.id}> has been updated... committing changes.",
+                fg="green",
+            )
+            record._record.commit()
+            # Step 2: Update the resource type in the draft
+            if draft._record["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            # After updating the record, update the draft's fork_version_id to match the record's new version_id, to avoid conflicts when publishing
+            draft._record.fork_version_id = record._record.revision_id
+            draft._record.commit()
+            # Commit the changes for both the record and the draft in one transaction
+            db.session.commit()
+            records_service.indexer.index(record._record)
+            records_service.indexer.index(draft._record)
+            secho(f"Draft <{draft.id}> has been updated successfully.", fg="green")
+            # Update DOI metadata if record has DOI
+            if hasattr(record, "pids") and record.pids.get("doi", None):
+                records_service.pids.register_or_update(
+                    system_identity, record.id, "doi", parent=False
+                )
+                secho(
+                    f"DOI metadata for record {record.id} has been updated successfully.",
+                    fg="green",
+                )
+        except DraftNotCreatedError:
+            # If the draft didn't exist, we simply edit and publish the record
+            draft = records_service.edit(system_identity, record.id)
+            if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft.data["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            updated_draft = records_service.update_draft(
+                system_identity, draft.id, draft.data
+            )
+            record = records_service.publish(system_identity, updated_draft.id)
+
+        secho(f"Record <{record.id}> has been updated successfully.", fg="green")
+
+    def migrate_resource_type_in_draft(draft_id):
+        """
+        Update resource type from publication-thesis to publication-dissertation.
+
+        We go through the service layer to automatically trigger the DOI update and re-indexing.
+        """
+        secho(f"Updating resource type for draft {draft_id}", fg="yellow")
+        draft = records_service.edit(system_identity, draft_id)
+        if draft.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
+            secho(
+                f"Skipping draft <{draft.id}> because it doesn't have resource-type 'publication-thesis'!",
+                fg="yellow",
+            )
+            return
+
+        if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+            draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
+        for related_identifier in draft.data["metadata"].get("related_identifiers", []):
+            if (
+                related_identifier.get("resource_type", {}).get("id")
+                == "publication-thesis"
+            ):
+                related_identifier["resource_type"]["id"] = "publication-dissertation"
+        updated_draft = records_service.update_draft(
+            system_identity, draft.id, draft.data
+        )
+        secho(f"Draft <{updated_draft.id}> has been updated successfully.", fg="green")
+
+    secho("Resource type update has started.", fg="green")
+
+    run_upgrade(
+        migrate_resource_type_in_record,
+        migrate_resource_type_in_draft,
+    )
+
+    secho("Resource type update has finished.", fg="green")
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 13.0 to 14.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    THIS MODULE IS WORK IN PROGRESS, UNTIL official v14 release
+
+    NOTE:
+    since the data upgrade steps are more selective now, the approach how to do
+    it has been changed. now the records/drafts which should be updated are
+    searched by a filter and then the updates are applied to those
+    records/drafts explicitly. this should improve speed and should make it
+    easier to upgrade large instances
+
+    """
+    secho("Starting data migration...", fg="green")
+
+    run_update_for_resource_type()
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_2_0_to_3_0.py

@@ -16,8 +16,11 @@ is that nothing happens!
 
 from invenio_db import db
 from invenio_rdm_records.records.api import RDMDraft, RDMParent, RDMRecord
-from invenio_rdm_records.records.models import RDMDraftMetadata, \
-    RDMFileDraftMetadata, RDMRecordMetadata
+from invenio_rdm_records.records.models import (
+    RDMDraftMetadata,
+    RDMFileDraftMetadata,
+    RDMRecordMetadata,
+)
 
 
 def execute_upgrade():
@@ -65,7 +68,8 @@ def execute_upgrade():
     # Cleanup associated deleted drafts.
 
     drafts = RDMDraftMetadata.query.filter(
-        RDMDraftMetadata.is_deleted == True).all()  # noqa
+        RDMDraftMetadata.is_deleted == True  # noqa
+    ).all()  # noqa
     for d in drafts:
         # Delete all file draft records
         RDMFileDraftMetadata.query.filter_by(record_id=d.id).delete()

invenio_app_rdm/upgrade_scripts/migrate_3_0_to_4_0.py

@@ -35,8 +35,8 @@ def execute_upgrade():
             languages_list = list(map(dict, unique_languages))
             record["metadata"]["languages"] = languages_list
 
-    def update_resource_type(record):
-        """Updates resource type to become a vocabulary."""
+    def update_vocabularies(record):
+        """Updates languages and resource_type to become vocabularies."""
 
         def get_res_type_vocabulary(data):
             """Returns the id value of the resource type vocabulary."""
@@ -45,18 +45,34 @@ def execute_upgrade():
             elif "type" in data["resource_type"]:
                 return data["resource_type"]["type"]
 
+        def get_language_vocabulary(data):
+            """Returns the language as vocabulary."""
+            return dict(id=data)
+
+        def migrate_language(field):
+            """Migrates language field."""
+            for idx, val in enumerate(record["metadata"].get(field, [])):
+                if "lang" in val:
+                    language_vocab = get_language_vocabulary(val["lang"])
+                    record["metadata"][field][idx]["lang"] = language_vocab
+
+        # Migrate resource_type
         if "resource_type" in record["metadata"]:
             res_type_vocab = get_res_type_vocabulary(record["metadata"])
             record["metadata"]["resource_type"] = dict(id=res_type_vocab)
 
-        for idx, val in enumerate(
-            record["metadata"].get("related_identifiers", [])
-        ):
+        # Migrate resource_type of related_identifiers
+        for idx, val in enumerate(record["metadata"].get("related_identifiers", [])):
             if "resource_type" in val:
                 res_type_vocab = get_res_type_vocabulary(val)
-                record["metadata"]["related_identifiers"][idx][
-                    "resource_type"
-                ] = dict(id=res_type_vocab)
+                record["metadata"]["related_identifiers"][idx]["resource_type"] = dict(
+                    id=res_type_vocab
+                )
+
+        # Migrate languages from additional_descriptions
+        migrate_language("additional_descriptions")
+        # Migrate languages from additional_titles
+        migrate_language("additional_titles")
 
         record.commit()
 
@@ -98,7 +114,7 @@ def execute_upgrade():
     for record_metadata in RDMRecord.model_cls.query.all():
         record = RDMRecord(record_metadata.data, model=record_metadata)
 
-        update_resource_type(record)
+        update_vocabularies(record)
 
     for draft_metadata in RDMDraft.model_cls.query.all():
         # Skipping deleted drafts because can't be committed
@@ -107,7 +123,7 @@ def execute_upgrade():
 
         draft = RDMDraft(draft_metadata.data, model=draft_metadata)
 
-        update_resource_type(draft)
+        update_vocabularies(draft)
 
     for parent_metadata in RDMParent.model_cls.query.all():
         parent = RDMParent(parent_metadata.data, model=parent_metadata)