invenio-app-rdm 8.0.0.dev9__py2.py3-none-any.whl → 14.0.0b4.dev4__py2.py3-none-any.whl

invenio_app_rdm/upgrade_scripts/fix_migrated_records_from_1_0_to_2_0.py

@@ -15,6 +15,7 @@ def execute_upgrade_fix():
 
     def update_siblings_versions(records_list):
         """Update records to assign a correct index."""
+
         def get_created_date(record):
             """Returns created date of the record."""
             return record.created

invenio_app_rdm/upgrade_scripts/fix_migrated_records_from_8_0_to_9_0.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+# Copyright (C) 2022 TU Wien.
+# Copyright (C) 2022 Graz University of Technology.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration fix script from InvenioRDM 8.0 to 9.0."""
+
+from click import secho
+from invenio_db import db
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+
+
+def execute_upgrade_fix():
+    """Execute the upgrade fix from InvenioRDM 8.0 to 9.0."""
+
+    def update_parent(record):
+        """Update parent schema and parent communities for older records."""
+        new_parent_schema = "local://records/parent-v2.0.0.json"
+        record.parent["$schema"] = new_parent_schema
+        record.parent.setdefault("communities", {})
+
+    def update_record(record):
+        # skipping deleted records because can't be committed
+        if record.is_deleted:
+            return
+
+        try:
+            secho(f"Updating record : {record.pid.pid_value}", fg="yellow")
+
+            update_parent(record)
+
+            record.parent.commit()
+            record.commit()
+
+            secho(f"> Updated parent: {record.parent.pid.pid_value}", fg="green")
+            secho(f"> Updated record: {record.pid.pid_value}\n", fg="green")
+            return None
+        except Exception as e:
+            secho("> Error {}".format(repr(e)), fg="red")
+            error = "Record {} failed to update".format(record.pid.pid_value)
+            return error
+
+    errors = []
+    for record_metadata in RDMRecord.model_cls.query.all():
+        record = RDMRecord(record_metadata.data, model=record_metadata)
+        error = update_record(record)
+        if error:
+            errors.append(error)
+
+    for draft_metadata in RDMDraft.model_cls.query.all():
+        draft = RDMDraft(draft_metadata.data, model=draft_metadata)
+        error = update_record(draft)
+        if error:
+            errors.append(error)
+
+    success = not errors
+
+    if success:
+        secho("Committing to DB", nl=True)
+        db.session.commit()
+        secho(
+            "Data migration completed, please rebuild the search indices now.",
+            fg="green",
+        )
+
+    else:
+        secho("Rollback", nl=True)
+        db.session.rollback()
+        secho(
+            "Upgrade aborted due to the following errors:",
+            fg="red",
+            err=True,
+        )
+
+        for error in errors:
+            secho(error, fg="red", err=True)
+
+        msg = (
+            "The changes have been rolled back. "
+            "Please fix the above listed errors and try the upgrade again",
+        )
+        secho(msg, fg="yellow", err=True)
+
+
+if __name__ == "__main__":
+    execute_upgrade_fix()

invenio_app_rdm/upgrade_scripts/migrate_10_0_to_11_0.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 10.0 to 11.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 10.0 to 11.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+"""
+
+from click import secho
+from invenio_db import db
+from invenio_files_rest.models import Bucket, FileInstance
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 10.0 to 11.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    """
+    secho("Starting data migration...", fg="green")
+    buckets = Bucket.query.all()
+
+    for bucket in buckets:
+        bucket.default_storage_class = "L"
+    db.session.add(bucket)
+
+    file_instances = FileInstance.query.all()
+
+    for fi in file_instances:
+        fi.storage_class = "L"
+        db.session.add(fi)
+
+    secho("Committing to DB", fg="green")
+    db.session.commit()
+    secho("Data migration completed.", fg="green")
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_11_0_to_12_0.py

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023-2024 CERN.
+# Copyright (C) 2024 Graz University of Technology.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 11.0 to 12.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 11.0 to 12.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested with following data:
+
+- user
+  - demo records of v11
+  - demo communities of v11
+  - cli created user of v11
+  - ui created community (com_a)
+  - ui created community private (com_b)
+  - ui created record (rec_a.v1)
+  - ui created v2 of record (rec_a.v1)
+  - ui created record (rec_b.v1) added to community (com_a)
+  - ui created draft (dra_a)
+  - ui created draft (dra_b) added to community (com_b)
+  - repository with records without managed doi
+  - repository with records with managed doi
+  - repository with records with managed doi without parent doi after migration
+  - base vocabularies (no customized) usable after migration
+  - record (rec_a.v1, rec_a.v2) findable after migration and rebuild of index
+
+- administration
+  - user panel list of users visible
+  - drafts visible
+  - records visible
+"""
+import sys
+
+from click import secho
+from flask import current_app
+from invenio_access.permissions import system_identity
+from invenio_communities.communities.records.api import Community
+from invenio_communities.communities.records.systemfields.access import ReviewPolicyEnum
+from invenio_db import db
+from invenio_rdm_records.fixtures import PrioritizedVocabulariesFixtures
+from invenio_rdm_records.proxies import current_rdm_records
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 11.0 to 12.0.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    """
+
+    def migrate_review_policy(community_record):
+        if community_record.is_deleted:
+            return
+
+        community_record["access"].setdefault(
+            "review_policy", ReviewPolicyEnum.CLOSED.value
+        )
+
+    def update_parent(record):
+        """Update parent schema and parent communities for older records."""
+        new_parent_schema = "local://records/parent-v3.0.0.json"
+        record.parent["$schema"] = new_parent_schema
+
+        if (
+            isinstance(record.parent["access"]["owned_by"], list)
+            and len(record.parent["access"]["owned_by"]) > 0
+        ):
+            record.parent.access.owned_by = {
+                "user": record.parent["access"]["owned_by"][0]["user"]
+            }
+
+        if "pids" not in record.parent:
+            record.parent["pids"] = {}
+
+            if (
+                current_app.config["DATACITE_ENABLED"]
+                and "doi" in current_app.config["RDM_PARENT_PERSISTENT_IDENTIFIERS"]
+                and current_app.config["RDM_PARENT_PERSISTENT_IDENTIFIERS"]["doi"][
+                    "is_enabled"
+                ]
+            ):
+                pids = current_rdm_records.records_service.pids.parent_pid_manager.create_all(
+                    record.parent, pids={}, schemes={"doi"}
+                )
+                current_rdm_records.records_service.pids.parent_pid_manager.reserve_all(
+                    record.parent, pids
+                )
+                record.parent["pids"] = pids
+                # Have to commit here otherwise register_or_update won't get
+                # the above data
+                record.parent.commit()
+
+                if isinstance(record, RDMRecord):
+                    current_rdm_records.records_service.pids.register_or_update(
+                        id_=record["id"],
+                        identity=system_identity,
+                        scheme="doi",
+                        parent=True,
+                    )
+
+    def update_record(record):
+        # skipping deleted records because can't be committed
+        if record.is_deleted:
+            return
+
+        try:
+            secho(f"Updating record : {record.pid.pid_value}", fg="yellow")
+
+            # otherwise the save would not work, due to new attributes
+            # (media_files, parent_doi) used
+            record["$schema"] = "local://records/record-v6.0.0.json"
+
+            # Initialize media files as disabled if not any
+            record.setdefault("media_files", {"enabled": False})
+            if record.media_files.bucket is None:
+                record.media_files.create_bucket()
+
+            update_parent(record)
+
+            record.commit()
+
+            secho(f"> Updated parent: {record.parent.pid.pid_value}", fg="green")
+            secho(f"> Updated record: {record.pid.pid_value}\n", fg="green")
+            return None
+        except Exception as e:
+            secho(f"> Error {repr(e)}", fg="red")
+            error = f"Record {record.pid.pid_value} failed to update"
+            return error
+
+    secho("Starting data migration...", fg="green")
+
+    # upgrading vocabularies
+    pvf = PrioritizedVocabulariesFixtures(system_identity)
+    pvf.load()
+
+    # Migrating communities
+    communities = Community.model_cls.query.all()
+
+    for community_data in communities:
+        community = Community(community_data.data, model=community_data)
+
+        # production data could have problems without it
+        if community:
+            migrate_review_policy(community)
+            community.commit()
+
+    # Migrating records and drafts
+    errors = []
+    for record_metadata in RDMRecord.model_cls.query.all():
+        record = RDMRecord(record_metadata.data, model=record_metadata)
+        error = update_record(record)
+
+        if error:
+            errors.append(error)
+
+    for draft_metadata in RDMDraft.model_cls.query.all():
+        draft = RDMDraft(draft_metadata.data, model=draft_metadata)
+        error = update_record(draft)
+        if error:
+            errors.append(error)
+
+    success = not errors
+
+    if success:
+        secho("Commiting to DB", nl=True)
+        db.session.commit()
+        secho(
+            "Data migration completed, please rebuild the search indices now.",
+            fg="green",
+        )
+
+    else:
+        secho("Rollback", nl=True)
+        db.session.rollback()
+        secho(
+            "Upgrade aborted due to the following errors:",
+            fg="red",
+            err=True,
+        )
+
+        for error in errors:
+            secho(error, fg="red", err=True)
+
+        msg = (
+            "The changes have been rolled back. "
+            "Please fix the above listed errors and try the upgrade again",
+        )
+        secho(msg, fg="yellow", err=True)
+
+        sys.exit(1)
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_12_0_to_13_0.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023-2025 CERN.
+# Copyright (C) 2024-2025 Graz University of Technology.
+# Copyright (C) 2025 Northwestern University.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 12.0 to 13.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 12.0 to 13.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested with following data:
+
+- record
+    - internal_notes
+"""
+
+import sys
+import traceback
+
+from click import secho
+from invenio_access.permissions import system_identity
+from invenio_db import db
+from invenio_rdm_records.proxies import current_rdm_records_service as records_service
+from invenio_search import current_search_client as search_client
+from invenio_search.engine import dsl
+from invenio_search.utils import prefix_index
+from invenio_vocabularies.contrib.affiliations.api import Affiliation
+from invenio_vocabularies.contrib.names.api import Name
+from sqlalchemy import select
+
+
+def run_upgrade(has, migrate):
+    """Run upgrade."""
+    record_success_counter = 0
+    record_error_counter = 0
+    draft_success_counter = 0
+    draft_error_counter = 0
+
+    # Handle published records
+    published_records = records_service.search(
+        system_identity,
+        params={"allversions": True, "include_deleted": True},
+        extra_filter=has,
+    )
+    for result in published_records.hits:
+        record = records_service.record_cls.pid.resolve(result["id"])
+        try:
+            migrate(record)
+            record_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Record {record.pid.pid_value} failed to update"
+            record_error_counter += 1
+
+    # Handle draft records
+    draft_records = records_service.search_drafts(
+        system_identity,
+        params={"allversions": True},
+        extra_filter=has,
+    )
+    for result in draft_records.hits:
+        draft = records_service.draft_cls.pid.resolve(
+            result["id"],
+            registered_only=False,
+        )
+        try:
+            migrate(draft)
+            draft_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Draft {draft.pid.pid_value} failed to update"
+            draft_error_counter += 1
+
+    if draft_error_counter > 0 or record_error_counter > 0:
+        db.session.rollback()
+        secho(
+            f"Migration failed: {record_error_counter} records had failures and {draft_error_counter} drafts had failures",
+            fg="red",
+        )
+        secho(
+            "The changes have been rolled back. Please fix the above listed errors and try the upgrade again",
+            fg="yellow",
+            err=True,
+        )
+    elif draft_success_counter > 0 or record_success_counter > 0:
+        db.session.commit()
+        secho(
+            f"Migration completed: {record_success_counter} records have been updated and {draft_success_counter} drafts have been updated",
+            fg="green",
+        )
+    else:
+        secho(
+            "Migration completed: no records or drafts required updating.", fg="green"
+        )
+
+
+def run_upgrade_for_thesis():
+    """Run upgrade for thesis."""
+
+    def migrate_thesis_university(record_or_draft):
+        custom_fields = record_or_draft.get("custom_fields", {})
+        university = custom_fields.get("thesis:university")
+        if university and "thesis:thesis" not in custom_fields:
+            custom_fields["thesis:thesis"] = {"university": university}
+
+        record_or_draft.commit()
+
+    # Common query filter
+    has_thesis = dsl.Q("exists", field="custom_fields.thesis:university")
+
+    secho("Thesis upgrade has started.", fg="green")
+
+    run_upgrade(has_thesis, migrate_thesis_university)
+
+    secho("Thesis upgrade has finished.", fg="green")
+
+
+def run_upgrade_for_affiliations():
+    """Update affiliations entry so that they conform to new shape."""
+    secho("Affiliations upgrade has started.", fg="green")
+
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Affiliation.model_cls).execution_options(yield_per=250)
+
+    for affiliation_model in db.session.scalars(stmt):
+        try:
+            data_for_affiliation = affiliation_model.data
+            data_for_affiliation.pop("id", None)
+            data_for_affiliation.pop("pid", None)
+            affiliation = Affiliation(data_for_affiliation, model=affiliation_model)
+            affiliation.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Affiliation {affiliation_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Affiliations upgrade failed.", fg="red")
+        sys.exit(1)
+    else:
+        db.session.commit()
+        secho("Affiliations upgrade succeeded.", fg="green")
+
+
+def run_upgrade_for_names():
+    """Update names entry so that they conform to new shape."""
+    secho("Names upgrade has started.", fg="green")
+
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Name.model_cls).execution_options(yield_per=250)
+
+    for name_model in db.session.scalars(stmt):
+        try:
+            data_for_name = name_model.data
+            data_for_name.pop("id", None)
+            data_for_name.pop("pid", None)
+            name = Name(data_for_name, model=name_model)
+            name.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Name {name_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Names upgrade failed.", fg="red")
+        sys.exit(1)
+    else:
+        db.session.commit()
+        secho("Names upgrade succeeded.", fg="green")
+
+
+def run_upgrade_for_event_stats_mappings():
+    """Update the live event stats mappings to add missing fields."""
+    secho("Event stats mappings upgrade has started.", fg="green")
+
+    # Find the latest mappings for views and download stats events
+    for event_type in ("record-view", "file-download"):
+        try:
+            events_index = prefix_index(f"events-stats-{event_type}-*")
+            res = search_client.indices.get(events_index)
+            last_two_indices = sorted(res.keys())[-2:]
+            for index in last_two_indices:
+                res = search_client.indices.put_mapping(
+                    index=index,
+                    body={"properties": {"is_machine": {"type": "boolean"}}},
+                )
+        except Exception as e:
+            secho(f"Mapping update for {event_type} failed with '{repr(e)}'.", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+    secho("Event stats mappings upgrade succeeded.", fg="green")
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 12.0 to 13.0.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+
+    NOTE:
+    since the data upgrade steps are more selective now, the approach how to do
+    it has been changed. now the records/drafts which should be updated are
+    searched by a filter and then the updates are applied to those
+    records/drafts explicitly. this should improve speed and should make it
+    easier to upgrade large instances
+
+    """
+    secho("Starting data migration...", fg="green")
+
+    run_upgrade_for_thesis()
+    run_upgrade_for_affiliations()
+    run_upgrade_for_names()
+    run_upgrade_for_event_stats_mappings()
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()