invenio-app-rdm 8.0.0.dev9__py2.py3-none-any.whl → 14.0.0b4.dev4__py2.py3-none-any.whl

invenio_app_rdm/upgrade_scripts/migrate_13_0_to_14_0.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 13.0 to 14.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 13.0 to 14.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested for the following scenarios:
+1. Draft with resource type publication-thesis
+2. Record with resource type publication-thesis with a DOI and no draft
+3. Record with resource type publication-thesis with a no DOI and an existing draft
+4. Records with multiple versions
+"""
+
+from click import secho
+from invenio_access.permissions import system_identity
+from invenio_db import db
+from invenio_drafts_resources.resources.records.errors import DraftNotCreatedError
+from invenio_rdm_records.proxies import current_rdm_records_service as records_service
+from invenio_search.api import RecordsSearchV2
+
+
+def run_upgrade(migrate_record, migrate_draft):
+    """Run upgrade on selected records and drafts.
+
+    Args:
+        migrate_record (callable): Function to migrate a record.
+        migrate_draft (callable): Function to migrate a draft.
+    """
+    errored_record_ids = []
+    errored_draft_ids = []
+
+    # Handle published records
+    published_records = (
+        RecordsSearchV2(index=records_service.record_cls.index._name)
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )  # Only need to fetch the record IDs to make the query faster
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    published_record_ids = [result["id"] for result in published_records]
+    for record_id in published_record_ids:
+        try:
+            migrate_record(record_id)
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            secho(f"Record {record_id} failed to update", fg="red")
+            errored_record_ids.append((record_id, error))
+
+    # Handle draft records
+    draft_records = (
+        RecordsSearchV2(index=records_service.draft_cls.index._name)
+        .filter("term", has_draft=False)
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    draft_record_ids = [result["id"] for result in draft_records]
+    for draft_id in draft_record_ids:
+        try:
+            migrate_draft(draft_id)
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            secho(f"Draft {draft_id} failed to update", fg="red")
+            errored_draft_ids.append((draft_id, error))
+
+    print(f"Errored record IDs:", *errored_record_ids, sep="\n")
+    print(f"Errored draft IDs:", *errored_draft_ids, sep="\n")
+
+
+def run_update_for_resource_type():
+    """Run update for resource type."""
+
+    def migrate_resource_type_in_record(record_id):
+        """
+        Update resource type from publication-thesis to publication-dissertation.
+
+        We go through the service layer to automatically trigger the DOI update and re-indexing.
+        """
+        secho(f"Updating resource type for record {record_id}", fg="yellow")
+        record = records_service.read(system_identity, record_id)
+        if record.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in record.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
+            secho(
+                f"Skipping record <{record.id}> because it doesn't have resource-type 'publication-thesis'!",
+                fg="yellow",
+            )
+            return
+
+        try:
+            draft = records_service.read_draft(system_identity, record.id)
+            # Step 1: Update the resource type in the record via low-level API
+            # We need to make sure we don't publish the record with different metadata
+            secho(
+                f"Record <{record.id}> has an existing draft <{draft.id}>! Updating record via low-level API.",
+                fg="yellow",
+            )
+            # Update the record directly without affecting the draft
+            if (
+                record._record["metadata"]["resource_type"]["id"]
+                == "publication-thesis"
+            ):
+                record._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in record._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            # Save the record changes and reindex
+            secho(
+                f"Record <{record.id}> has been updated... committing changes.",
+                fg="green",
+            )
+            record._record.commit()
+            # Step 2: Update the resource type in the draft
+            if draft._record["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            # After updating the record, update the draft's fork_version_id to match the record's new version_id, to avoid conflicts when publishing
+            draft._record.fork_version_id = record._record.revision_id
+            draft._record.commit()
+            # Commit the changes for both the record and the draft in one transaction
+            db.session.commit()
+            records_service.indexer.index(record._record)
+            records_service.indexer.index(draft._record)
+            secho(f"Draft <{draft.id}> has been updated successfully.", fg="green")
+            # Update DOI metadata if record has DOI
+            if hasattr(record, "pids") and record.pids.get("doi", None):
+                records_service.pids.register_or_update(
+                    system_identity, record.id, "doi", parent=False
+                )
+                secho(
+                    f"DOI metadata for record {record.id} has been updated successfully.",
+                    fg="green",
+                )
+        except DraftNotCreatedError:
+            # If the draft didn't exist, we simply edit and publish the record
+            draft = records_service.edit(system_identity, record.id)
+            if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft.data["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            updated_draft = records_service.update_draft(
+                system_identity, draft.id, draft.data
+            )
+            record = records_service.publish(system_identity, updated_draft.id)
+
+        secho(f"Record <{record.id}> has been updated successfully.", fg="green")
+
+    def migrate_resource_type_in_draft(draft_id):
+        """
+        Update resource type from publication-thesis to publication-dissertation.
+
+        We go through the service layer to automatically trigger the DOI update and re-indexing.
+        """
+        secho(f"Updating resource type for draft {draft_id}", fg="yellow")
+        draft = records_service.edit(system_identity, draft_id)
+        if draft.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
+            secho(
+                f"Skipping draft <{draft.id}> because it doesn't have resource-type 'publication-thesis'!",
+                fg="yellow",
+            )
+            return
+
+        if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+            draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
+        for related_identifier in draft.data["metadata"].get("related_identifiers", []):
+            if (
+                related_identifier.get("resource_type", {}).get("id")
+                == "publication-thesis"
+            ):
+                related_identifier["resource_type"]["id"] = "publication-dissertation"
+        updated_draft = records_service.update_draft(
+            system_identity, draft.id, draft.data
+        )
+        secho(f"Draft <{updated_draft.id}> has been updated successfully.", fg="green")
+
+    secho("Resource type update has started.", fg="green")
+
+    run_upgrade(
+        migrate_resource_type_in_record,
+        migrate_resource_type_in_draft,
+    )
+
+    secho("Resource type update has finished.", fg="green")
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 13.0 to 14.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    THIS MODULE IS WORK IN PROGRESS, UNTIL official v14 release
+
+    NOTE:
+    since the data upgrade steps are more selective now, the approach how to do
+    it has been changed. now the records/drafts which should be updated are
+    searched by a filter and then the updates are applied to those
+    records/drafts explicitly. this should improve speed and should make it
+    easier to upgrade large instances
+
+    """
+    secho("Starting data migration...", fg="green")
+
+    run_update_for_resource_type()
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_2_0_to_3_0.py

@@ -16,8 +16,11 @@ is that nothing happens!
 
 from invenio_db import db
 from invenio_rdm_records.records.api import RDMDraft, RDMParent, RDMRecord
-from invenio_rdm_records.records.models import RDMDraftMetadata, \
-    RDMFileDraftMetadata, RDMRecordMetadata
+from invenio_rdm_records.records.models import (
+    RDMDraftMetadata,
+    RDMFileDraftMetadata,
+    RDMRecordMetadata,
+)
 
 
 def execute_upgrade():
@@ -65,7 +68,8 @@ def execute_upgrade():
     # Cleanup associated deleted drafts.
 
     drafts = RDMDraftMetadata.query.filter(
-        RDMDraftMetadata.is_deleted == True).all()  # noqa
+        RDMDraftMetadata.is_deleted == True  # noqa
+    ).all()  # noqa
     for d in drafts:
         # Delete all file draft records
         RDMFileDraftMetadata.query.filter_by(record_id=d.id).delete()

invenio_app_rdm/upgrade_scripts/migrate_3_0_to_4_0.py

@@ -62,14 +62,12 @@ def execute_upgrade():
             record["metadata"]["resource_type"] = dict(id=res_type_vocab)
 
         # Migrate resource_type of related_identifiers
-        for idx, val in enumerate(
-            record["metadata"].get("related_identifiers", [])
-        ):
+        for idx, val in enumerate(record["metadata"].get("related_identifiers", [])):
             if "resource_type" in val:
                 res_type_vocab = get_res_type_vocabulary(val)
-                record["metadata"]["related_identifiers"][idx][
-                    "resource_type"
-                ] = dict(id=res_type_vocab)
+                record["metadata"]["related_identifiers"][idx]["resource_type"] = dict(
+                    id=res_type_vocab
+                )
 
         # Migrate languages from additional_descriptions
         migrate_language("additional_descriptions")

invenio_app_rdm/upgrade_scripts/migrate_4_0_to_6_0.py

@@ -15,13 +15,11 @@ is that nothing happens!
 
 import yaml
 from flask import current_app
-from invenio_access.permissions import system_identity
 from invenio_db import db
 from invenio_pidstore.models import PersistentIdentifier
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
 from invenio_vocabularies.records.api import Vocabulary
-from invenio_vocabularies.records.models import VocabularyScheme, \
-    VocabularyType
+from invenio_vocabularies.records.models import VocabularyScheme, VocabularyType
 from sqlalchemy.orm.exc import NoResultFound
 
 
@@ -49,8 +47,7 @@ def migrate_vocabularies():
 
     # remove old affiliations
     try:
-        schemes = VocabularyScheme.query.filter_by(
-            parent_id="affiliations").all()
+        schemes = VocabularyScheme.query.filter_by(parent_id="affiliations").all()
         for scheme in schemes:
             db.session.delete(scheme)
         db.session.commit()
@@ -58,8 +55,7 @@ def migrate_vocabularies():
     except NoResultFound:
         print("No affiliations scheme not found: OK.")
     try:
-        db.session.delete(VocabularyType.query.filter_by(
-            id="affiliations").one())
+        db.session.delete(VocabularyType.query.filter_by(id="affiliations").one())
         db.session.commit()
         print("Affiliations vocabulary type removed: OK.")
     except NoResultFound:
@@ -110,7 +106,7 @@ def check_affiliations():
         if inv_affs_creators or inv_affs_contributors:
             invalid_affiliations_rec[record["id"]] = {
                 "creators": inv_affs_creators,
-                "contributors": inv_affs_contributors
+                "contributors": inv_affs_contributors,
             }
 
     invalid_affiliations_draft = {}
@@ -130,7 +126,7 @@ def check_affiliations():
             invalid_affiliations_draft[draft["id"]] = {
                 "type": "draft",
                 "creators": inv_affs_creators,
-                "contributors": inv_affs_contributors
+                "contributors": inv_affs_contributors,
             }
 
     invalid_affiliations = {}
@@ -141,17 +137,17 @@ def check_affiliations():
 
     if invalid_affiliations:
         print(
-            f"Your instance has {total} affiliations that need to be " +
-            "fixed. Check the invalid_affiliation.yaml file for more details."
+            f"Your instance has {total} affiliations that need to be "
+            + "fixed. Check the invalid_affiliation.yaml file for more details."
         )
-        with open('invalid_affiliations.yaml', 'w') as f:
+        with open("invalid_affiliations.yaml", "w") as f:
             yaml.dump(list(invalid_affiliations), f)
     else:
-        print(f"All your instance's affiliations are valid.")
+        print("All your instance's affiliations are valid.")
     if needs_ror:
         print(
-            "You have affiliations with ROR identifiers, you need to " +
-            "add its vocabulary. Instructions to do so are available in "
+            "You have affiliations with ROR identifiers, you need to "
+            + "add its vocabulary. Instructions to do so are available in "
             "https://inveniordm.docs.cern.ch/customize/vocabularies/affiliations/"  # noqa
         )
 
@@ -170,7 +166,7 @@ def check_subjects():
                 vocab_subjects[id_] = {
                     "id": id_,
                     "scheme": subject["scheme"],
-                    "subject": subject["subject"]
+                    "subject": subject["subject"],
                 }
 
         return vocab_subjects
@@ -189,21 +185,18 @@ def check_subjects():
         record = RDMRecord(record_metadata.data, model=record_metadata)
         # publish record subjects take presedence if id is repeated
         # | operator is only available from py 3.9 on
-        subjects_to_dump = {
-            **subjects_to_dump,
-            **_should_be_vocabulary(record)
-        }
+        subjects_to_dump = {**subjects_to_dump, **_should_be_vocabulary(record)}
 
     total = len(subjects_to_dump)
     if subjects_to_dump:
         print(
-            f"Your instance has {total} subjects that " +
-            "should be custom vocabularies."
+            f"Your instance has {total} subjects that "
+            + "should be custom vocabularies."
         )
-        with open('custom_subjects.yaml', 'w') as f:
+        with open("custom_subjects.yaml", "w") as f:
             yaml.dump(list(subjects_to_dump.values()), f)
     else:
-        print(f"All your instance's subjects are valid.")
+        print("All your instance's subjects are valid.")
 
 
 def execute_upgrade():
@@ -212,6 +205,7 @@ def execute_upgrade():
     Please read the disclaimer on this module before thinking about executing
     this function!
     """
+
     def update_roles(creatibutors):
         """Update roles."""
         for creatibutor in creatibutors:
@@ -328,20 +322,15 @@ def execute_upgrade():
         """Update contributors roles."""
         contributors = record.get("metadata").get("contributors", [])
         if contributors:
-            record["metadata"]["contributors"] = \
-                update_affiliations(contributors)
+            record["metadata"]["contributors"] = update_affiliations(contributors)
 
     def update_rights(record):
         """Update record rights."""
         rights = record.get("metadata").get("rights", [])
         for right in rights:
-            locale = current_app.config.get('BABEL_DEFAULT_LOCALE', 'en')
-            right["title"] = {
-                locale: right["title"]
-            }
-            right["description"] = {
-                locale: right["description"]
-            }
+            locale = current_app.config.get("BABEL_DEFAULT_LOCALE", "en")
+            right["title"] = {locale: right["title"]}
+            right["description"] = {locale: right["description"]}
 
     def migrate_record(record):
         """Migrates a record/draft to the new schema's values."""
@@ -351,9 +340,7 @@ def execute_upgrade():
         update_contributors_roles(record)
         update_additional_titles(record)
         update_additional_descriptions(record)
-        update_list_field_vocabulary(
-            record, "related_identifiers", "relation_type"
-        )
+        update_list_field_vocabulary(record, "related_identifiers", "relation_type")
         update_list_field_vocabulary(record, "dates", "type")
         update_subjects(record)
         update_creators_affiliations(record)
@@ -384,13 +371,13 @@ def execute_upgrade():
 
 if __name__ == "__main__":
     print(
-        "Choose a step:\n" +
-        "[1] Migrate old vocabularies\n" +
-        "[2] Check record's subjects\n" +
-        "[3] Check creators/contributors affiliations\n" +
-        "[4] Migrate records\n" +
-        ">> ",
-        end=""
+        "Choose a step:\n"
+        + "[1] Migrate old vocabularies\n"
+        + "[2] Check record's subjects\n"
+        + "[3] Check creators/contributors affiliations\n"
+        + "[4] Migrate records\n"
+        + ">> ",
+        end="",
     )
     try:
         choice = int(input())

invenio_app_rdm/upgrade_scripts/migrate_6_0_to_7_0.py

@@ -13,8 +13,6 @@ If this script is executed at any other time, probably the best case scenario
 is that nothing happens!
 """
 
-import sys
-
 from click import secho
 from invenio_db import db
 from invenio_rdm_records.proxies import current_rdm_records_service
@@ -146,9 +144,7 @@ def execute_upgrade():
 
         except Exception as exc:
             success = False
-            secho(
-                f"Error while generating OAI PIDs: {exc}\n", fg="red", err=True
-            )
+            secho(f"Error while generating OAI PIDs: {exc}\n", fg="red", err=True)
 
     if not success:
         secho(

invenio_app_rdm/upgrade_scripts/migrate_7_0_to_8_0.py

@@ -19,8 +19,10 @@ import invenio_rdm_records.fixtures as fixtures
 from click import echo, secho
 from invenio_access.permissions import system_identity
 from invenio_db import db
-from invenio_rdm_records.fixtures import PrioritizedVocabulariesFixtures, \
-    VocabulariesFixture
+from invenio_rdm_records.fixtures import (
+    PrioritizedVocabulariesFixtures,
+    VocabulariesFixture,
+)
 from invenio_vocabularies.proxies import current_service as vocabulary_svc
 
 
@@ -88,9 +90,7 @@ def execute_upgrade():
     # 1: check the app_data directory for the invenio instance
     app_data_yaml_file = pf._app_data_folder / pf._filename
     if app_data_yaml_file.exists():
-        app_data_fixture = VocabulariesFixture(
-            system_identity, app_data_yaml_file
-        )
+        app_data_fixture = VocabulariesFixture(system_identity, app_data_yaml_file)
         licenses_loaded, errs = update_licenses_from_fixture(app_data_fixture)
         errors.extend(errs)
 
@@ -104,17 +104,13 @@ def execute_upgrade():
         # check if the entry points define something for licenses
         if not licenses_loaded and "licenses" in vocab_list:
             extension_fixture = VocabulariesFixture(system_identity, filepath)
-            licenses_loaded, errs = update_licenses_from_fixture(
-                extension_fixture
-            )
+            licenses_loaded, errs = update_licenses_from_fixture(extension_fixture)
             errors.extend(errs)
 
     # 3: check the default vocabularies from rdm-records
     pkg_data_yaml_file = pf._pkg_data_folder / pf._filename
     if not licenses_loaded and pkg_data_yaml_file.exists():
-        pkg_data_fixture = VocabulariesFixture(
-            system_identity, pkg_data_yaml_file
-        )
+        pkg_data_fixture = VocabulariesFixture(system_identity, pkg_data_yaml_file)
         licenses_loaded, errs = update_licenses_from_fixture(pkg_data_fixture)
         errors.extend(errs)
 

invenio_app_rdm/upgrade_scripts/migrate_8_0_to_9_0.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 8.0 to 9.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 8.0 to 9.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+"""
+
+from click import echo, secho
+from invenio_db import db
+from invenio_rdm_records.records.api import RDMRecord
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 8.0 to 9.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    """
+
+    def update_funding_field(record):
+        try:
+            echo(f"Updating record: {record['id']}... ", nl=False)
+            for funding in record.metadata.get("funding", []):
+                award = funding.get("award", {})
+                funder = funding.get("funder", {})
+                if award.get("identifier") and award.get("scheme"):
+                    award_identifier = award.pop("identifier")
+                    award_scheme = award.pop("scheme")
+                    funding["award"]["identifiers"] = [
+                        {"identifier": award_identifier, "scheme": award_scheme}
+                    ]
+                if funder.get("identifier") and funder.get("scheme"):
+                    funder.pop("identifier")
+                    funder.pop("scheme")
+                if award.get("title", ""):
+                    award_title = award.pop("title")
+                    funding["award"]["title"] = {"en": award_title}
+                if funder.get("title", ""):
+                    funder.pop("title")
+            secho("OK", fg="green")
+            return record
+        except Exception as e:
+            secho("Error {}".format(repr(e)), fg="red")
+            return None
+
+    errors = []
+
+    records = RDMRecord.model_cls.query.all()
+    for record in records:
+        r = RDMRecord(record.data, model=record)
+        r["$schema"] = "local://records/record-v5.0.0.json"
+        res = update_funding_field(r)
+        if res is None:
+            errors.append("Record {} failed to update funding".format(r.id))
+        else:
+            r = res
+            r.commit()
+
+    success = not errors
+
+    if success:
+        echo("Committing to DB", nl=True)
+        db.session.commit()
+        secho(
+            "Data migration completed, please rebuild the search indices now.",
+            fg="green",
+        )
+
+    else:
+        echo("Rollback", nl=True)
+        db.session.rollback()
+        secho(
+            "Upgrade aborted due to the following errors:",
+            fg="red",
+            err=True,
+        )
+
+        for error in errors:
+            secho(error, fg="red", err=True)
+
+        msg = (
+            "The changes have been rolled back. "
+            "Please fix the above listed errors and try the upgrade again",
+        )
+        secho(msg, fg="yellow", err=True)
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()