invenio-app-rdm 13.0.0b4.dev1__py2.py3-none-any.whl → 14.0.0b1.dev8__py2.py3-none-any.whl

invenio_app_rdm/upgrade_scripts/migrate_12_0_to_13_0.py

@@ -1,15 +1,16 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2023-2024 CERN.
-# Copyright (C) 2024 Graz University of Technology.
+# Copyright (C) 2023-2025 CERN.
+# Copyright (C) 2024-2025 Graz University of Technology.
+# Copyright (C) 2025 Northwestern University.
 #
 # Invenio-App-RDM is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
 
-"""Record migration script from InvenioRDM 11.0 to 12.0.
+"""Record migration script from InvenioRDM 12.0 to 13.0.
 
 Disclaimer: This script is intended to be executed *only once*, namely when
-upgrading from InvenioRDM 11.0 to 12.0!
+upgrading from InvenioRDM 12.0 to 13.0!
 If this script is executed at any other time, probably the best case scenario
 is that nothing happens!
 
@@ -21,85 +22,212 @@ This script has been tested with following data:
 """
 
 import sys
+import traceback
 
 from click import secho
+from invenio_access.permissions import system_identity
 from invenio_db import db
-from invenio_rdm_records.records.api import RDMDraft, RDMRecord
-
+from invenio_rdm_records.proxies import current_rdm_records_service as records_service
+from invenio_search import current_search_client as search_client
+from invenio_search.engine import dsl
+from invenio_search.utils import prefix_index
+from invenio_vocabularies.contrib.affiliations.api import Affiliation
+from invenio_vocabularies.contrib.names.api import Name
+from sqlalchemy import select
+
+
+def run_upgrade(has, migrate):
+    """Run upgrade."""
+    record_success_counter = 0
+    record_error_counter = 0
+    draft_success_counter = 0
+    draft_error_counter = 0
+
+    # Handle published records
+    published_records = records_service.search(
+        system_identity,
+        params={"allversions": True, "include_deleted": True},
+        extra_filter=has,
+    )
+    for result in published_records.hits:
+        record = records_service.record_cls.pid.resolve(result["id"])
+        try:
+            migrate(record)
+            record_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Record {record.pid.pid_value} failed to update"
+            record_error_counter += 1
+
+    # Handle draft records
+    draft_records = records_service.search_drafts(
+        system_identity,
+        params={"allversions": True},
+        extra_filter=has,
+    )
+    for result in draft_records.hits:
+        draft = records_service.draft_cls.pid.resolve(
+            result["id"],
+            registered_only=False,
+        )
+        try:
+            migrate(draft)
+            draft_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Draft {draft.pid.pid_value} failed to update"
+            draft_error_counter += 1
+
+    if draft_error_counter > 0 or record_error_counter > 0:
+        db.session.rollback()
+        secho(
+            f"Migration failed: {record_error_counter} records had failures and {draft_error_counter} drafts had failures",
+            fg="red",
+        )
+        secho(
+            "The changes have been rolled back. Please fix the above listed errors and try the upgrade again",
+            fg="yellow",
+            err=True,
+        )
+    elif draft_success_counter > 0 or record_success_counter > 0:
+        db.session.commit()
+        secho(
+            f"Migration completed: {record_success_counter} records have been updated and {draft_success_counter} drafts have been updated",
+            fg="green",
+        )
+    else:
+        secho(
+            "Migration completed: no records or drafts required updating.", fg="green"
+        )
 
-def execute_upgrade():
-    """Execute the upgrade from InvenioRDM 12.0 to 13.0.0.
 
-    Please read the disclaimer on this module before thinking about executing
-    this function!
-    THIS MODULE IS WORK IN PROGRESS, UNTIL official v13 release
-    """
+def run_upgrade_for_thesis():
+    """Run upgrade for thesis."""
 
-    def update_record(record):
-        # skipping deleted records because can't be committed
-        if record.is_deleted:
-            return
+    def migrate_thesis_university(record_or_draft):
+        custom_fields = record_or_draft.get("custom_fields", {})
+        university = custom_fields.get("thesis:university")
+        if university and "thesis:thesis" not in custom_fields:
+            custom_fields["thesis:thesis"] = {"university": university}
 
-        try:
-            secho(f"Updating record : {record.pid.pid_value}", fg="yellow")
+        record_or_draft.commit()
 
-            # TODO: Add any record datamodel migration code here
-            record.commit()
+    # Common query filter
+    has_thesis = dsl.Q("exists", field="custom_fields.thesis:university")
 
-            secho(f"> Updated parent: {record.parent.pid.pid_value}", fg="green")
-            secho(f"> Updated record: {record.pid.pid_value}\n", fg="green")
-            return None
-        except Exception as e:
-            secho(f"> Error {repr(e)}", fg="red")
-            error = f"Record {record.pid.pid_value} failed to update"
-            return error
+    secho("Thesis upgrade has started.", fg="green")
 
-    secho("Starting data migration...", fg="green")
+    run_upgrade(has_thesis, migrate_thesis_university)
 
-    # Migrating records and drafts
-    errors = []
-    for record_metadata in RDMRecord.model_cls.query.all():
-        record = RDMRecord(record_metadata.data, model=record_metadata)
-        error = update_record(record)
+    secho("Thesis upgrade has finished.", fg="green")
 
-        if error:
-            errors.append(error)
 
-    for draft_metadata in RDMDraft.model_cls.query.all():
-        draft = RDMDraft(draft_metadata.data, model=draft_metadata)
-        error = update_record(draft)
-        if error:
-            errors.append(error)
+def run_upgrade_for_affiliations():
+    """Update affiliations entry so that they conform to new shape."""
+    secho("Affiliations upgrade has started.", fg="green")
 
-    success = not errors
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Affiliation.model_cls).execution_options(yield_per=250)
 
-    if success:
-        secho("Commiting to DB", nl=True)
+    for affiliation_model in db.session.scalars(stmt):
+        try:
+            data_for_affiliation = affiliation_model.data
+            data_for_affiliation.pop("id", None)
+            data_for_affiliation.pop("pid", None)
+            affiliation = Affiliation(data_for_affiliation, model=affiliation_model)
+            affiliation.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Affiliation {affiliation_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Affiliations upgrade failed.", fg="red")
+        sys.exit(1)
+    else:
         db.session.commit()
-        secho(
-            "Data migration completed, please rebuild the search indices now.",
-            fg="green",
-        )
+        secho("Affiliations upgrade succeeded.", fg="green")
 
-    else:
-        secho("Rollback", nl=True)
-        db.session.rollback()
-        secho(
-            "Upgrade aborted due to the following errors:",
-            fg="red",
-            err=True,
-        )
 
-        for error in errors:
-            secho(error, fg="red", err=True)
+def run_upgrade_for_names():
+    """Update names entry so that they conform to new shape."""
+    secho("Names upgrade has started.", fg="green")
 
-        msg = (
-            "The changes have been rolled back. "
-            "Please fix the above listed errors and try the upgrade again",
-        )
-        secho(msg, fg="yellow", err=True)
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Name.model_cls).execution_options(yield_per=250)
 
+    for name_model in db.session.scalars(stmt):
+        try:
+            data_for_name = name_model.data
+            data_for_name.pop("id", None)
+            data_for_name.pop("pid", None)
+            name = Name(data_for_name, model=name_model)
+            name.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Name {name_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Names upgrade failed.", fg="red")
         sys.exit(1)
+    else:
+        db.session.commit()
+        secho("Names upgrade succeeded.", fg="green")
+
+
+def run_upgrade_for_event_stats_mappings():
+    """Update the live event stats mappings to add missing fields."""
+    secho("Event stats mappings upgrade has started.", fg="green")
+
+    # Find the latest mappings for views and download stats events
+    for event_type in ("record-view", "file-download"):
+        try:
+            events_index = prefix_index(f"events-stats-{event_type}-*")
+            res = search_client.indices.get(events_index)
+            last_two_indices = sorted(res.keys())[-2:]
+            for index in last_two_indices:
+                res = search_client.indices.put_mapping(
+                    index=index,
+                    body={"properties": {"is_machine": {"type": "boolean"}}},
+                )
+        except Exception as e:
+            secho(f"Mapping update for {event_type} failed with '{repr(e)}'.", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+    secho("Event stats mappings upgrade succeeded.", fg="green")
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 12.0 to 13.0.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+
+    NOTE:
+    since the data upgrade steps are more selective now, the approach how to do
+    it has been changed. now the records/drafts which should be updated are
+    searched by a filter and then the updates are applied to those
+    records/drafts explicitly. this should improve speed and should make it
+    easier to upgrade large instances
+
+    """
+    secho("Starting data migration...", fg="green")
+
+    run_upgrade_for_thesis()
+    run_upgrade_for_affiliations()
+    run_upgrade_for_names()
+    run_upgrade_for_event_stats_mappings()
 
 
 # if the script is executed on its own, perform the upgrade

{invenio_app_rdm-13.0.0b4.dev1.dist-info → invenio_app_rdm-14.0.0b1.dev8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-app-rdm
-Version: 13.0.0b4.dev1
+Version: 14.0.0b1.dev8
 Summary: Invenio Research Data Management.
 Home-page: https://github.com/inveniosoftware/invenio-app-rdm
 Author: CERN
@@ -40,13 +40,13 @@ Requires-Dist: invenio-search-ui<5.0.0,>=4.0.0
 Requires-Dist: invenio-files-rest<4.0.0,>=3.0.0
 Requires-Dist: invenio-previewer<4.0.0,>=3.0.0
 Requires-Dist: invenio-records-files<2.0.0,>=1.2.1
-Requires-Dist: invenio-collections<1.0.0,>=0.4.0
-Requires-Dist: invenio-communities<20.0.0,>=19.0.0
-Requires-Dist: invenio-rdm-records<20.0.0,>=19.0.0
+Requires-Dist: invenio-collections<3.0.0,>=1.0.0
+Requires-Dist: invenio-communities<22.0.0,>=21.0.0
+Requires-Dist: invenio-rdm-records<22.0.0,>=21.0.0
 Requires-Dist: CairoSVG<3.0.0,>=2.5.2
 Requires-Dist: invenio-banners<6.0.0,>=5.0.0
 Requires-Dist: invenio-pages<8.0.0,>=7.0.0
-Requires-Dist: invenio-audit-logs<1.0.0,>=0.1.0
+Requires-Dist: invenio-audit-logs<1.0.0,>=0.3.0
 Requires-Dist: invenio-sitemap<2.0.0,>=0.1.0
 Provides-Extra: tests
 Requires-Dist: pytest-black-ng>=0.4.0; extra == "tests"
@@ -60,7 +60,7 @@ Requires-Dist: invenio-search[opensearch1]<4.0.0,>=3.0.0; extra == "opensearch1"
 Provides-Extra: opensearch2
 Requires-Dist: invenio-search[opensearch2]<4.0.0,>=3.0.0; extra == "opensearch2"
 Provides-Extra: s3
-Requires-Dist: invenio-s3<3.0.0,>=2.0.0; extra == "s3"
+Requires-Dist: invenio-s3<4.0.0,>=3.0.0; extra == "s3"
 Dynamic: license-file
 
 ..
@@ -90,7 +90,7 @@ https://inveniordm.docs.cern.ch
 
 
 ..
-    Copyright (C) 2019-2024 CERN.
+    Copyright (C) 2019-2025 CERN.
     Copyright (C) 2019-2024 Northwestern University.
     Copyright (C) 2021-2024 TU Wien.
     Copyright (C) 2021-2025 Graz University of Technology.
@@ -101,6 +101,117 @@ https://inveniordm.docs.cern.ch
 Changes
 =======
 
+Version v14.0.0b1.dev8 (released 2025-11-21)
+
+- UI: add info message about remaining days to publish changes
+- request: add file mod request template
+- fix: set files_locked to .locked rather than permission
+- feat: pass file modification eval to deposit form
+
+Version v14.0.0b1.dev7 (released 2025-11-14)
+
+- fix(views): pass API record to evaluate record deletion
+
+Version v14.0.0b1.dev6 (released 2025-11-13)
+
+- tombstone: add deletion policy
+- fix(deletion-request): never disable the existing deletion request link
+- fix(deletion-request): fetch existing request only for valid users
+- fix(deletion-request): removal reasons in landing page data attribute
+- vocab: add deletion_request prefix to removal_reasons vocabulary
+- ui: added related identifiers to vocabulary
+- views: remove unused data attributes from recordManagementMobile
+- requests: ui: only show justification if present
+
+Version v14.0.0b1.dev5 (released 2025-10-14)
+
+- fix(community-submission): add missing var in side bar template
+- fix(guest-access-request): handle broken expiration date payload value
+    * this extra check prevents the request timeline UI from breaking when a
+      request payload has a broken value, to make recovery easier
+    * also, make eslint happy with a camel case variable name
+- fix(guest-access-request): handle more possible expiration date values
+    * when the expiration date gets submitted, the value of the empty input
+      field is typically the empty string rather than `null`
+    * on the reading side, handle the case that the expiration date is set
+      as a string value (since request payloads are generally strings)
+
+Version v14.0.0b1.dev4 (released 2025-10-03)
+
+- installation: relax invenio-collections dependency requirements
+    * see https://github.com/inveniosoftware/invenio-app-rdm/issues/3194
+
+Version v14.0.0b1.dev3 (released 2025-10-01)
+
+- fix(deposit-ui): read expanded published record
+    * The UI serializer requires the expanded record service result.
+- feat(config): add last activity sort option
+
+Version v14.0.0b1.dev2 (released 2025-09-29)
+
+- fix(deposit-ui): use API record for evaluating deletion policy
+
+Version v14.0.0b1.dev1 (released 2025-09-24)
+
+- Fixed CHANGES.rst formatting
+
+Version v14.0.0b1.dev0 (released 2025-09-24)
+
+- feat(administration): add Requests view
+- feat: add record deletion requests
+    * Allows to configure record deletion policies for immediate deletion
+      or going through a request flow on the record landing page.
+
+Version v14.0.0b0.dev4 (released 2025-09-01)
+
+- views(deposit): pass to the form config the published record if we edit one
+    * The published record is used to compare published DOI but it can be
+      used also to compare with the published state of an editing draft.
+- help: correct search instructions for missing fields
+
+Version v14.0.0b0.dev3 (released 2025-08-26)
+
+- fix(community-requests): request permissions were missing from UI
+    * The community requests page was not showing the request reviewers section
+      because the permissions were not passed to the template.
+
+Version v14.0.0b0.dev2 (released 2025-08-08)
+
+- fix(views): use correct community value
+    * After the change in 33f1b0d2, values passed to the UI or used for
+      accessing UI-related attributes (e.g. the "theme"), must come from
+      the community UI-serialized value instead of the service result item.
+- fix(pages): add community base template as a valid option
+- feat(fixtures): allow specifying `template_name` in page fixtures
+- ui: refactor record injecting in JS components
+    - inject only record links in citations cmps
+    - remove dependency between citations and communities cmps which
+      they were sharing the same record injection
+- fix(deposits): pass missing community_ui to community theme templates
+    * also introduce more explicit distinctions between community and
+      community_ui variables
+- UI: set order of entries and rename category to deposits
+- UI: make external resources section consistent with details
+- UI: make rights section consistent with details
+
+Version v14.0.0b0.dev1 (released 2025-08-01)
+
+- setup: bump invenio-communities and invenio-rdm-records
+- theme: add search overrides for reviewers search results
+- requests: add request action_accept permissions to UI
+- theme: update package-lock.json for compiling catalog for frontend tests
+
+Version v13.0.0b4.dev2 (released 2025-06-13)
+
+- checks: integrate into community requests flow
+- deposit-ui: pass draft errors to form
+- landing page: text-break references
+- ui: fixed caret trigger in stats
+
+- deposit: report actual file quota rather than configured values
+    * previously, the deposit form would show values from the configuration
+      that might be different from the actually effective quota
+
 
 Version v13.0.0b4.dev1 (released 2025-06-10)