invenio-app-rdm 14.0.0b2.dev1__py2.py3-none-any.whl → 14.0.0b2.dev2__py2.py3-none-any.whl

invenio_app_rdm/__init__.py

@@ -17,6 +17,6 @@
 #
 # See PEP 0440 for details - https://www.python.org/dev/peps/pep-0440
 
-__version__ = "14.0.0b2.dev1"
+__version__ = "14.0.0b2.dev2"
 
 __all__ = ("__version__",)

invenio_app_rdm/upgrade_scripts/migrate_13_0_to_14_0.py

@@ -20,68 +20,91 @@ This script has been tested for the following scenarios:
 4. Records with multiple versions
 """
 
-import time
-
 from click import secho
 from invenio_access.permissions import system_identity
 from invenio_db import db
 from invenio_drafts_resources.resources.records.errors import DraftNotCreatedError
 from invenio_rdm_records.proxies import current_rdm_records_service as records_service
-from invenio_search.engine import dsl
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+from invenio_search.api import RecordsSearchV2
 
 
-def run_upgrade(has, migrate_record, migrate_draft):
+def run_upgrade(migrate_record, migrate_draft):
     """Run upgrade on selected records and drafts.
 
     Args:
-        has (dsl.Q): Query filter to select records/drafts to update.
         migrate_record (callable): Function to migrate a record.
         migrate_draft (callable): Function to migrate a draft.
     """
+    errored_record_ids = []
+    errored_draft_ids = []
+
     # Handle published records
-    published_records = records_service.scan(
-        identity=system_identity,
-        params={"allversions": True},
-        extra_filter=has,
-    )
-    for result in published_records.hits:
+    published_records = (
+        RecordsSearchV2(index=records_service.record_cls.index._name)
+        .filter("term", deletion_status="P")
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )  # Only need to fetch the record IDs to make the query faster
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    published_record_ids = [result["id"] for result in published_records]
+    for record_id in published_record_ids:
         try:
-            migrate_record(result)
+            migrate_record(record_id)
         except Exception as error:
             secho(f"> Error {repr(error)}", fg="red")
-            error = f"Record {result['id']} failed to update"
+            secho(f"Record {record_id} failed to update", fg="red")
+            errored_record_ids.append((record_id, error))
 
     # Handle draft records
-    draft_records = records_service._search(
-        identity=system_identity,
-        action="scan",
-        params={"allversions": True},
-        search_preference=None,
-        record_cls=records_service.draft_cls,
-        search_opts=records_service.config.search_drafts,
-        extra_filter=has,
-        permission_action="read_draft",
-    ).scan()
-    for result in draft_records:
+    draft_records = (
+        RecordsSearchV2(index=records_service.draft_cls.index._name)
+        .filter("term", has_draft=False)
+        .filter(
+            "query_string",
+            query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis",
+        )
+        .source(["id"])
+        .scan()
+    )
+    # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes
+    draft_record_ids = [result["id"] for result in draft_records]
+    for draft_id in draft_record_ids:
         try:
-            migrate_draft(result)
+            migrate_draft(draft_id)
         except Exception as error:
             secho(f"> Error {repr(error)}", fg="red")
-            error = f"Draft {result['id']} failed to update"
+            secho(f"Draft {draft_id} failed to update", fg="red")
+            errored_draft_ids.append((draft_id, error))
+
+    print(f"Errored record IDs:", *errored_record_ids, sep="\n")
+    print(f"Errored draft IDs:", *errored_draft_ids, sep="\n")
 
 
 def run_update_for_resource_type():
     """Run update for resource type."""
 
-    def migrate_resource_type_in_record(hit_result):
+    def migrate_resource_type_in_record(record_id):
         """
         Update resource type from publication-thesis to publication-dissertation.
 
         We go through the service layer to automatically trigger the DOI update and re-indexing.
         """
-        secho(f"Updating resource type for record {hit_result['id']}", fg="yellow")
-        record = records_service.read(system_identity, hit_result["id"])
-        if record.data["metadata"]["resource_type"]["id"] != "publication-thesis":
+        secho(f"Updating resource type for record {record_id}", fg="yellow")
+        record = records_service.read(system_identity, record_id)
+        if record.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in record.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
             secho(
                 f"Skipping record <{record.id}> because it doesn't have resource-type 'publication-thesis'!",
                 fg="yellow",
@@ -97,31 +120,78 @@ def run_update_for_resource_type():
                 fg="yellow",
             )
             # Update the record directly without affecting the draft
-            record._record["metadata"]["resource_type"][
-                "id"
-            ] = "publication-dissertation"
+            if (
+                record._record["metadata"]["resource_type"]["id"]
+                == "publication-thesis"
+            ):
+                record._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in record._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
             # Save the record changes and reindex
+            secho(
+                f"Record <{record.id}> has been updated... committing changes.",
+                fg="green",
+            )
             record._record.commit()
+            # Step 2: Update the resource type in the draft
+            if draft._record["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft._record["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft._record["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
+            # After updating the record, update the draft's fork_version_id to match the record's new version_id, to avoid conflicts when publishing
+            draft._record.fork_version_id = record._record.revision_id
+            draft._record.commit()
+            # Commit the changes for both the record and the draft in one transaction
             db.session.commit()
             records_service.indexer.index(record._record)
+            records_service.indexer.index(draft._record)
+            secho(f"Draft <{draft.id}> has been updated successfully.", fg="green")
             # Update DOI metadata if record has DOI
             if hasattr(record, "pids") and record.pids.get("doi", None):
                 records_service.pids.register_or_update(
                     system_identity, record.id, "doi", parent=False
                 )
-            # Step 2: Update the resource type in the draft
-            secho(f"Updating resource type for draft {draft.id}", fg="yellow")
-            draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
-            # After updating the record, update the draft's fork_version_id to match the record's new version_id, to avoid conflicts when publishing
-            draft._record.fork_version_id = record._record.revision_id
-            updated_draft = records_service.update_draft(
-                system_identity, draft.id, draft.data
-            )
-            secho(f"Draft {draft.id} has been updated successfully.", fg="green")
+                secho(
+                    f"DOI metadata for record {record.id} has been updated successfully.",
+                    fg="green",
+                )
         except DraftNotCreatedError:
             # If the draft didn't exist, we simply edit and publish the record
             draft = records_service.edit(system_identity, record.id)
-            draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
+            if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+                draft.data["metadata"]["resource_type"][
+                    "id"
+                ] = "publication-dissertation"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            ):
+                if (
+                    related_identifier.get("resource_type", {}).get("id")
+                    == "publication-thesis"
+                ):
+                    related_identifier["resource_type"][
+                        "id"
+                    ] = "publication-dissertation"
             updated_draft = records_service.update_draft(
                 system_identity, draft.id, draft.data
             )
@@ -129,36 +199,45 @@ def run_update_for_resource_type():
 
         secho(f"Record <{record.id}> has been updated successfully.", fg="green")
 
-    def migrate_resource_type_in_draft(hit_result):
+    def migrate_resource_type_in_draft(draft_id):
         """
         Update resource type from publication-thesis to publication-dissertation.
 
         We go through the service layer to automatically trigger the DOI update and re-indexing.
         """
-        secho(f"Updating resource type for draft {hit_result['id']}", fg="yellow")
-        draft = records_service.edit(system_identity, hit_result["id"])
-        if draft.data["metadata"]["resource_type"]["id"] != "publication-thesis":
+        secho(f"Updating resource type for draft {draft_id}", fg="yellow")
+        draft = records_service.edit(system_identity, draft_id)
+        if draft.data["metadata"]["resource_type"][
+            "id"
+        ] != "publication-thesis" and not any(
+            related_identifier.get("resource_type", {}).get("id")
+            == "publication-thesis"
+            for related_identifier in draft.data["metadata"].get(
+                "related_identifiers", []
+            )
+        ):
             secho(
                 f"Skipping draft <{draft.id}> because it doesn't have resource-type 'publication-thesis'!",
                 fg="yellow",
             )
             return
 
-        draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
+        if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis":
+            draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation"
+        for related_identifier in draft.data["metadata"].get("related_identifiers", []):
+            if (
+                related_identifier.get("resource_type", {}).get("id")
+                == "publication-thesis"
+            ):
+                related_identifier["resource_type"]["id"] = "publication-dissertation"
         updated_draft = records_service.update_draft(
             system_identity, draft.id, draft.data
         )
         secho(f"Draft <{updated_draft.id}> has been updated successfully.", fg="green")
 
-    # Query records/drafts with resource type publication-thesis
-    has_resource_type = dsl.Q(
-        "query_string", query="metadata.resource_type.id:publication-thesis"
-    )
-
     secho("Resource type update has started.", fg="green")
 
     run_upgrade(
-        has_resource_type,
         migrate_resource_type_in_record,
         migrate_resource_type_in_draft,
     )

{invenio_app_rdm-14.0.0b2.dev1.dist-info → invenio_app_rdm-14.0.0b2.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-app-rdm
-Version: 14.0.0b2.dev1
+Version: 14.0.0b2.dev2
 Summary: Invenio Research Data Management.
 Home-page: https://github.com/inveniosoftware/invenio-app-rdm
 Author: CERN
@@ -101,6 +101,10 @@ https://inveniordm.docs.cern.ch
 Changes
 =======
 
+Version v14.0.0b2.dev2 (released 2025-10-24)
+
+- upgrade_scripts: v14: optimize scan to avoid scroll context overhead
+
 Version v14.0.0b2.dev1 (released 2025-10-21)
 
 - deposit-form: updated related works options vocab

{invenio_app_rdm-14.0.0b2.dev1.dist-info → invenio_app_rdm-14.0.0b2.dev2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-invenio_app_rdm/__init__.py,sha256=YKw26EOYsN7GAX40s95XuSugzggJs6SM10HE0xQ7P-s,704
+invenio_app_rdm/__init__.py,sha256=_vvW0mLU8A2mkHzDXnANLUM4PlYFMn68veC_cQZKYCs,704
 invenio_app_rdm/cli.py,sha256=G6QqNU2W6n6ICtTMnpeKFXIsdorncDmVXwwwsGH5F2k,2746
 invenio_app_rdm/config.py,sha256=kxAliLtzXLRysO8qxWxoSt1FhOjuDfzUi8xr40BBIYE,53632
 invenio_app_rdm/ext.py,sha256=K7syn5CU5If7yOclFeNOCZX_u5q6VB7NJEQVm41mlng,5286
@@ -475,7 +475,7 @@ invenio_app_rdm/upgrade_scripts/fix_migrated_records_from_8_0_to_9_0.py,sha256=p
 invenio_app_rdm/upgrade_scripts/migrate_10_0_to_11_0.py,sha256=TX6FCWXY4qM4z7IYzDO5qaMTheo3zAjFrmR1sXaEf4U,1333
 invenio_app_rdm/upgrade_scripts/migrate_11_0_to_12_0.py,sha256=Tp7jfT2JHrYCFzF2qIYqG7yr7k-GhX2zkw61CWJGA78,6941
 invenio_app_rdm/upgrade_scripts/migrate_12_0_to_13_0.py,sha256=pyO68jyGyKXVTcja8tpi2XgNx_FxXk7JhgDTV-wx3xM,8205
-invenio_app_rdm/upgrade_scripts/migrate_13_0_to_14_0.py,sha256=TGq3Sfze6Sn97d45TloD7iyNHuaLK-O8O8tXJJiJytQ,7715
+invenio_app_rdm/upgrade_scripts/migrate_13_0_to_14_0.py,sha256=-JyKmXQolkeh0dUKdEekVhVGTYZ4AncyjZdy4n5DIeQ,11356
 invenio_app_rdm/upgrade_scripts/migrate_1_0_records_to_2_0.py,sha256=mRDv_Ao5zMgA6X0aogMfvhspO1CIApKtDW_ziJp5fjI,3325
 invenio_app_rdm/upgrade_scripts/migrate_2_0_to_3_0.py,sha256=jL_2I61Q9qt3fjBzYYueeT4EMQ9FlNPxYE4nzDQbLEY,2698
 invenio_app_rdm/upgrade_scripts/migrate_3_0_to_4_0.py,sha256=BNjGufwLBvLHnu0gz5b_Are-FuxYjXlCtkLgNQckV3U,4768
@@ -496,9 +496,9 @@ invenio_app_rdm/users_ui/views/__init__.py,sha256=SMdY2NJj9GICfr3Xuok7qdNYVtA2bJ
 invenio_app_rdm/users_ui/views/dashboard.py,sha256=iUn2PrODAwb8ugmMosJKAjPhUzjCiWiAWoXQr9RUFuc,1793
 invenio_app_rdm/users_ui/views/ui.py,sha256=W_eXM8dLVIrNHQB2UEh37C9BYoHauft6RyvcDNFHovA,1742
 invenio_app_rdm/utils/files.py,sha256=CruDyO2gDVadSlWEJD-WHpWHeOQ0juh-Ei9jz3D9yjc,3923
-invenio_app_rdm-14.0.0b2.dev1.dist-info/licenses/LICENSE,sha256=AZXFHRrZa5s4m9DV7zZr4bPGTMUvcEPCodeV_AmFI8k,1204
-invenio_app_rdm-14.0.0b2.dev1.dist-info/METADATA,sha256=ythPvg1U9pqyZyITAJJTZPqMDJQ3_GB5HSNqvaNvcuA,19345
-invenio_app_rdm-14.0.0b2.dev1.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-invenio_app_rdm-14.0.0b2.dev1.dist-info/entry_points.txt,sha256=MwtT1SN5saWOgTYhNb5y0YGA9VGAi0kXN0cykIfsb4U,2405
-invenio_app_rdm-14.0.0b2.dev1.dist-info/top_level.txt,sha256=quZejDUw2vLfKQboNIuVLJ9fxZifdnCT_s2PNf1dfmk,16
-invenio_app_rdm-14.0.0b2.dev1.dist-info/RECORD,,
+invenio_app_rdm-14.0.0b2.dev2.dist-info/licenses/LICENSE,sha256=AZXFHRrZa5s4m9DV7zZr4bPGTMUvcEPCodeV_AmFI8k,1204
+invenio_app_rdm-14.0.0b2.dev2.dist-info/METADATA,sha256=0hlSvkdwVNB33JLqW1cdDKSQ75MdsYT0wjIsERoreJY,19463
+invenio_app_rdm-14.0.0b2.dev2.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+invenio_app_rdm-14.0.0b2.dev2.dist-info/entry_points.txt,sha256=MwtT1SN5saWOgTYhNb5y0YGA9VGAi0kXN0cykIfsb4U,2405
+invenio_app_rdm-14.0.0b2.dev2.dist-info/top_level.txt,sha256=quZejDUw2vLfKQboNIuVLJ9fxZifdnCT_s2PNf1dfmk,16
+invenio_app_rdm-14.0.0b2.dev2.dist-info/RECORD,,