invenio-app-rdm 12.0.0.dev8__py2.py3-none-any.whl → 14.0.0b1.dev8__py2.py3-none-any.whl

invenio_app_rdm/upgrade_scripts/fix_migrated_records_from_8_0_to_9_0.py

@@ -60,7 +60,7 @@ def execute_upgrade_fix():
     success = not errors
 
     if success:
-        secho(f"Commiting to DB", nl=True)
+        secho("Committing to DB", nl=True)
         db.session.commit()
         secho(
             "Data migration completed, please rebuild the search indices now.",
@@ -68,7 +68,7 @@ def execute_upgrade_fix():
         )
 
     else:
-        secho(f"Rollback", nl=True)
+        secho("Rollback", nl=True)
         db.session.rollback()
         secho(
             "Upgrade aborted due to the following errors:",

invenio_app_rdm/upgrade_scripts/migrate_10_0_to_11_0.py

@@ -37,7 +37,7 @@ def execute_upgrade():
         fi.storage_class = "L"
         db.session.add(fi)
 
-    secho(f"Commiting to DB", fg="green")
+    secho("Committing to DB", fg="green")
     db.session.commit()
     secho("Data migration completed.", fg="green")
 

invenio_app_rdm/upgrade_scripts/migrate_11_0_to_12_0.py

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023-2024 CERN.
+# Copyright (C) 2024 Graz University of Technology.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 11.0 to 12.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 11.0 to 12.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested with following data:
+
+- user
+  - demo records of v11
+  - demo communities of v11
+  - cli created user of v11
+  - ui created community (com_a)
+  - ui created community private (com_b)
+  - ui created record (rec_a.v1)
+  - ui created v2 of record (rec_a.v1)
+  - ui created record (rec_b.v1) added to community (com_a)
+  - ui created draft (dra_a)
+  - ui created draft (dra_b) added to community (com_b)
+  - repository with records without managed doi
+  - repository with records with managed doi
+  - repository with records with managed doi without parent doi after migration
+  - base vocabularies (no customized) usable after migration
+  - record (rec_a.v1, rec_a.v2) findable after migration and rebuild of index
+
+- administration
+  - user panel list of users visible
+  - drafts visible
+  - records visible
+"""
+import sys
+
+from click import secho
+from flask import current_app
+from invenio_access.permissions import system_identity
+from invenio_communities.communities.records.api import Community
+from invenio_communities.communities.records.systemfields.access import ReviewPolicyEnum
+from invenio_db import db
+from invenio_rdm_records.fixtures import PrioritizedVocabulariesFixtures
+from invenio_rdm_records.proxies import current_rdm_records
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 11.0 to 12.0.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+    """
+
+    def migrate_review_policy(community_record):
+        if community_record.is_deleted:
+            return
+
+        community_record["access"].setdefault(
+            "review_policy", ReviewPolicyEnum.CLOSED.value
+        )
+
+    def update_parent(record):
+        """Update parent schema and parent communities for older records."""
+        new_parent_schema = "local://records/parent-v3.0.0.json"
+        record.parent["$schema"] = new_parent_schema
+
+        if (
+            isinstance(record.parent["access"]["owned_by"], list)
+            and len(record.parent["access"]["owned_by"]) > 0
+        ):
+            record.parent.access.owned_by = {
+                "user": record.parent["access"]["owned_by"][0]["user"]
+            }
+
+        if "pids" not in record.parent:
+            record.parent["pids"] = {}
+
+            if (
+                current_app.config["DATACITE_ENABLED"]
+                and "doi" in current_app.config["RDM_PARENT_PERSISTENT_IDENTIFIERS"]
+                and current_app.config["RDM_PARENT_PERSISTENT_IDENTIFIERS"]["doi"][
+                    "is_enabled"
+                ]
+            ):
+                pids = current_rdm_records.records_service.pids.parent_pid_manager.create_all(
+                    record.parent, pids={}, schemes={"doi"}
+                )
+                current_rdm_records.records_service.pids.parent_pid_manager.reserve_all(
+                    record.parent, pids
+                )
+                record.parent["pids"] = pids
+                # Have to commit here otherwise register_or_update won't get
+                # the above data
+                record.parent.commit()
+
+                if isinstance(record, RDMRecord):
+                    current_rdm_records.records_service.pids.register_or_update(
+                        id_=record["id"],
+                        identity=system_identity,
+                        scheme="doi",
+                        parent=True,
+                    )
+
+    def update_record(record):
+        # skipping deleted records because can't be committed
+        if record.is_deleted:
+            return
+
+        try:
+            secho(f"Updating record : {record.pid.pid_value}", fg="yellow")
+
+            # otherwise the save would not work, due to new attributes
+            # (media_files, parent_doi) used
+            record["$schema"] = "local://records/record-v6.0.0.json"
+
+            # Initialize media files as disabled if not any
+            record.setdefault("media_files", {"enabled": False})
+            if record.media_files.bucket is None:
+                record.media_files.create_bucket()
+
+            update_parent(record)
+
+            record.commit()
+
+            secho(f"> Updated parent: {record.parent.pid.pid_value}", fg="green")
+            secho(f"> Updated record: {record.pid.pid_value}\n", fg="green")
+            return None
+        except Exception as e:
+            secho(f"> Error {repr(e)}", fg="red")
+            error = f"Record {record.pid.pid_value} failed to update"
+            return error
+
+    secho("Starting data migration...", fg="green")
+
+    # upgrading vocabularies
+    pvf = PrioritizedVocabulariesFixtures(system_identity)
+    pvf.load()
+
+    # Migrating communities
+    communities = Community.model_cls.query.all()
+
+    for community_data in communities:
+        community = Community(community_data.data, model=community_data)
+
+        # production data could have problems without it
+        if community:
+            migrate_review_policy(community)
+            community.commit()
+
+    # Migrating records and drafts
+    errors = []
+    for record_metadata in RDMRecord.model_cls.query.all():
+        record = RDMRecord(record_metadata.data, model=record_metadata)
+        error = update_record(record)
+
+        if error:
+            errors.append(error)
+
+    for draft_metadata in RDMDraft.model_cls.query.all():
+        draft = RDMDraft(draft_metadata.data, model=draft_metadata)
+        error = update_record(draft)
+        if error:
+            errors.append(error)
+
+    success = not errors
+
+    if success:
+        secho("Commiting to DB", nl=True)
+        db.session.commit()
+        secho(
+            "Data migration completed, please rebuild the search indices now.",
+            fg="green",
+        )
+
+    else:
+        secho("Rollback", nl=True)
+        db.session.rollback()
+        secho(
+            "Upgrade aborted due to the following errors:",
+            fg="red",
+            err=True,
+        )
+
+        for error in errors:
+            secho(error, fg="red", err=True)
+
+        msg = (
+            "The changes have been rolled back. "
+            "Please fix the above listed errors and try the upgrade again",
+        )
+        secho(msg, fg="yellow", err=True)
+
+        sys.exit(1)
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_12_0_to_13_0.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023-2025 CERN.
+# Copyright (C) 2024-2025 Graz University of Technology.
+# Copyright (C) 2025 Northwestern University.
+#
+# Invenio-App-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Record migration script from InvenioRDM 12.0 to 13.0.
+
+Disclaimer: This script is intended to be executed *only once*, namely when
+upgrading from InvenioRDM 12.0 to 13.0!
+If this script is executed at any other time, probably the best case scenario
+is that nothing happens!
+
+
+This script has been tested with following data:
+
+- record
+    - internal_notes
+"""
+
+import sys
+import traceback
+
+from click import secho
+from invenio_access.permissions import system_identity
+from invenio_db import db
+from invenio_rdm_records.proxies import current_rdm_records_service as records_service
+from invenio_search import current_search_client as search_client
+from invenio_search.engine import dsl
+from invenio_search.utils import prefix_index
+from invenio_vocabularies.contrib.affiliations.api import Affiliation
+from invenio_vocabularies.contrib.names.api import Name
+from sqlalchemy import select
+
+
+def run_upgrade(has, migrate):
+    """Run upgrade."""
+    record_success_counter = 0
+    record_error_counter = 0
+    draft_success_counter = 0
+    draft_error_counter = 0
+
+    # Handle published records
+    published_records = records_service.search(
+        system_identity,
+        params={"allversions": True, "include_deleted": True},
+        extra_filter=has,
+    )
+    for result in published_records.hits:
+        record = records_service.record_cls.pid.resolve(result["id"])
+        try:
+            migrate(record)
+            record_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Record {record.pid.pid_value} failed to update"
+            record_error_counter += 1
+
+    # Handle draft records
+    draft_records = records_service.search_drafts(
+        system_identity,
+        params={"allversions": True},
+        extra_filter=has,
+    )
+    for result in draft_records.hits:
+        draft = records_service.draft_cls.pid.resolve(
+            result["id"],
+            registered_only=False,
+        )
+        try:
+            migrate(draft)
+            draft_success_counter += 1
+        except Exception as error:
+            secho(f"> Error {repr(error)}", fg="red")
+            error = f"Draft {draft.pid.pid_value} failed to update"
+            draft_error_counter += 1
+
+    if draft_error_counter > 0 or record_error_counter > 0:
+        db.session.rollback()
+        secho(
+            f"Migration failed: {record_error_counter} records had failures and {draft_error_counter} drafts had failures",
+            fg="red",
+        )
+        secho(
+            "The changes have been rolled back. Please fix the above listed errors and try the upgrade again",
+            fg="yellow",
+            err=True,
+        )
+    elif draft_success_counter > 0 or record_success_counter > 0:
+        db.session.commit()
+        secho(
+            f"Migration completed: {record_success_counter} records have been updated and {draft_success_counter} drafts have been updated",
+            fg="green",
+        )
+    else:
+        secho(
+            "Migration completed: no records or drafts required updating.", fg="green"
+        )
+
+
+def run_upgrade_for_thesis():
+    """Run upgrade for thesis."""
+
+    def migrate_thesis_university(record_or_draft):
+        custom_fields = record_or_draft.get("custom_fields", {})
+        university = custom_fields.get("thesis:university")
+        if university and "thesis:thesis" not in custom_fields:
+            custom_fields["thesis:thesis"] = {"university": university}
+
+        record_or_draft.commit()
+
+    # Common query filter
+    has_thesis = dsl.Q("exists", field="custom_fields.thesis:university")
+
+    secho("Thesis upgrade has started.", fg="green")
+
+    run_upgrade(has_thesis, migrate_thesis_university)
+
+    secho("Thesis upgrade has finished.", fg="green")
+
+
+def run_upgrade_for_affiliations():
+    """Update affiliations entry so that they conform to new shape."""
+    secho("Affiliations upgrade has started.", fg="green")
+
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Affiliation.model_cls).execution_options(yield_per=250)
+
+    for affiliation_model in db.session.scalars(stmt):
+        try:
+            data_for_affiliation = affiliation_model.data
+            data_for_affiliation.pop("id", None)
+            data_for_affiliation.pop("pid", None)
+            affiliation = Affiliation(data_for_affiliation, model=affiliation_model)
+            affiliation.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Affiliation {affiliation_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Affiliations upgrade failed.", fg="red")
+        sys.exit(1)
+    else:
+        db.session.commit()
+        secho("Affiliations upgrade succeeded.", fg="green")
+
+
+def run_upgrade_for_names():
+    """Update names entry so that they conform to new shape."""
+    secho("Names upgrade has started.", fg="green")
+
+    error = False
+    # Batch intake to limit memory usage
+    stmt = select(Name.model_cls).execution_options(yield_per=250)
+
+    for name_model in db.session.scalars(stmt):
+        try:
+            data_for_name = name_model.data
+            data_for_name.pop("id", None)
+            data_for_name.pop("pid", None)
+            name = Name(data_for_name, model=name_model)
+            name.commit()
+        except Exception as e:
+            secho(f"Migration failed with '{repr(e)}'.", fg="red")
+            secho(f"Name {name_model.pid} failed to update", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+            error = True
+            break
+
+    if error:
+        db.session.rollback()
+        secho("Names upgrade failed.", fg="red")
+        sys.exit(1)
+    else:
+        db.session.commit()
+        secho("Names upgrade succeeded.", fg="green")
+
+
+def run_upgrade_for_event_stats_mappings():
+    """Update the live event stats mappings to add missing fields."""
+    secho("Event stats mappings upgrade has started.", fg="green")
+
+    # Find the latest mappings for views and download stats events
+    for event_type in ("record-view", "file-download"):
+        try:
+            events_index = prefix_index(f"events-stats-{event_type}-*")
+            res = search_client.indices.get(events_index)
+            last_two_indices = sorted(res.keys())[-2:]
+            for index in last_two_indices:
+                res = search_client.indices.put_mapping(
+                    index=index,
+                    body={"properties": {"is_machine": {"type": "boolean"}}},
+                )
+        except Exception as e:
+            secho(f"Mapping update for {event_type} failed with '{repr(e)}'.", fg="red")
+            trace = traceback.format_exc()
+            secho(f"Traceback {trace}", fg="red")
+    secho("Event stats mappings upgrade succeeded.", fg="green")
+
+
+def execute_upgrade():
+    """Execute the upgrade from InvenioRDM 12.0 to 13.0.0.
+
+    Please read the disclaimer on this module before thinking about executing
+    this function!
+
+    NOTE:
+    since the data upgrade steps are more selective now, the approach how to do
+    it has been changed. now the records/drafts which should be updated are
+    searched by a filter and then the updates are applied to those
+    records/drafts explicitly. this should improve speed and should make it
+    easier to upgrade large instances
+
+    """
+    secho("Starting data migration...", fg="green")
+
+    run_upgrade_for_thesis()
+    run_upgrade_for_affiliations()
+    run_upgrade_for_names()
+    run_upgrade_for_event_stats_mappings()
+
+
+# if the script is executed on its own, perform the upgrade
+if __name__ == "__main__":
+    execute_upgrade()

invenio_app_rdm/upgrade_scripts/migrate_2_0_to_3_0.py

@@ -68,7 +68,7 @@ def execute_upgrade():
     # Cleanup associated deleted drafts.
 
     drafts = RDMDraftMetadata.query.filter(
-        RDMDraftMetadata.is_deleted == True
+        RDMDraftMetadata.is_deleted == True  # noqa
     ).all()  # noqa
     for d in drafts:
         # Delete all file draft records

invenio_app_rdm/upgrade_scripts/migrate_4_0_to_6_0.py

@@ -15,7 +15,6 @@ is that nothing happens!
 
 import yaml
 from flask import current_app
-from invenio_access.permissions import system_identity
 from invenio_db import db
 from invenio_pidstore.models import PersistentIdentifier
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
@@ -144,7 +143,7 @@ def check_affiliations():
         with open("invalid_affiliations.yaml", "w") as f:
             yaml.dump(list(invalid_affiliations), f)
     else:
-        print(f"All your instance's affiliations are valid.")
+        print("All your instance's affiliations are valid.")
     if needs_ror:
         print(
             "You have affiliations with ROR identifiers, you need to "
@@ -197,7 +196,7 @@ def check_subjects():
         with open("custom_subjects.yaml", "w") as f:
             yaml.dump(list(subjects_to_dump.values()), f)
     else:
-        print(f"All your instance's subjects are valid.")
+        print("All your instance's subjects are valid.")
 
 
 def execute_upgrade():

invenio_app_rdm/upgrade_scripts/migrate_6_0_to_7_0.py

@@ -13,8 +13,6 @@ If this script is executed at any other time, probably the best case scenario
 is that nothing happens!
 """
 
-import sys
-
 from click import secho
 from invenio_db import db
 from invenio_rdm_records.proxies import current_rdm_records_service

invenio_app_rdm/upgrade_scripts/migrate_8_0_to_9_0.py

@@ -67,7 +67,7 @@ def execute_upgrade():
     success = not errors
 
     if success:
-        echo(f"Commiting to DB", nl=True)
+        echo("Committing to DB", nl=True)
         db.session.commit()
         secho(
             "Data migration completed, please rebuild the search indices now.",
@@ -75,7 +75,7 @@ def execute_upgrade():
         )
 
     else:
-        echo(f"Rollback", nl=True)
+        echo("Rollback", nl=True)
         db.session.rollback()
         secho(
             "Upgrade aborted due to the following errors:",

invenio_app_rdm/upgrade_scripts/migrate_9_0_to_10_0.py

@@ -30,7 +30,7 @@ def execute_upgrade():
         echo(f"Updating record: {system_created_set.id}... ", nl=True)
         system_created_set.system_created = True
 
-    echo(f"Commiting to DB", nl=True)
+    echo("Committing to DB", nl=True)
     db.session.commit()
     secho(
         "Data migration completed.",

invenio_app_rdm/users/schemas.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 Graz University of Technology.
+#
+# Invenio App RDM is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""RDM User Schemas."""
+
+from invenio_users_resources.services.schemas import (
+    NotificationPreferences,
+    UserPreferencesSchema,
+    UserSchema,
+)
+from marshmallow import fields
+
+
+class UserPreferencesNotificationsSchema(UserPreferencesSchema):
+    """Schema extending preferences with notification preferences for model validation."""
+
+    notifications = fields.Nested(NotificationPreferences)
+
+
+class NotificationsUserSchema(UserSchema):
+    """Schema extending preferences with notification preferences for user service."""
+
+    preferences = fields.Nested(UserPreferencesNotificationsSchema)

invenio_app_rdm/users_ui/searchapp.py

@@ -23,6 +23,10 @@ def search_app_context():
             current_app.config["RDM_SORT_OPTIONS"],
             "/api/user/records",
             {"Accept": "application/vnd.inveniordm.v1+json"},
+            pagination_options=(10, 20),
+            initial_filters=[
+                ["shared_with_me", "false"],
+            ],
         ),
         "search_app_rdm_user_communities_config": partial(
             search_app_config,
@@ -31,6 +35,7 @@ def search_app_context():
             current_app.config["RDM_SORT_OPTIONS"],
             "/api/user/communities",
             {"Accept": "application/vnd.inveniordm.v1+json"},
+            pagination_options=(10, 20),
         ),
         "search_app_rdm_user_requests_config": partial(
             search_app_config,
@@ -39,7 +44,10 @@ def search_app_context():
             current_app.config["RDM_SORT_OPTIONS"],
             "/api/user/requests",
             {"Accept": "application/json"},
-            initial_filters=[["is_open", "true"]],
+            initial_filters=[
+                ["is_open", "true"],
+                ["shared_with_me", "false"],
+            ],
             hidden_params=[["expand", "1"]],
         ),
     }

invenio_app_rdm/users_ui/templates/semantic-ui/invenio_app_rdm/users/communities.html

@@ -19,6 +19,7 @@
   <div
     id="invenio-search-config"
     data-invenio-search-config='{{ search_app_rdm_user_communities_config(app_id="InvenioAppRdm.DashboardCommunities") | tojson }}'
+    data-can-create-community='{{ can_create_community | tojson }}'
   >
   </div>
 {%- endblock user_dashboard_body %}

invenio_app_rdm/users_ui/templates/semantic-ui/invenio_app_rdm/users/header.html

@@ -1,32 +1,46 @@
 {# -*- coding: utf-8 -*-
 
   This file is part of Invenio.
-  Copyright (C) 2022 CERN.
+  Copyright (C) 2022-2024 CERN.
 
   Invenio is free software; you can redistribute it and/or modify it
   under the terms of the MIT License; see LICENSE file for more details.
 #}
 
-<div class="ui container fluid page-subheader-outer shadowless with-submenu ml-0-mobile mr-0-mobile mb-0 rel-pt-2" id="dashboard-user-header-container">
-  <div class="ui container page-subheader">
-    <div class="ui items unstackable">
+<div class="ui container fluid page-subheader-outer shadowless with-submenu ml-0-mobile mr-0-mobile mb-0 pb-0 rel-pt-2" id="dashboard-user-header-container">
+  <div class="ui container page-subheader flex justify-space-between align-items-center">
+    <div class="ui items unstackable m-0">
       <div class="item">
         <div class="ui image dashboard-header-avatar">
           <img src="{{ user_avatar }}" alt="" />
         </div>
-        <div class="content rel-pl-1">
-          <h4 class="header">{{ current_userprofile.full_name or current_user.username }}</h4>
-          <div class="meta">{{ current_userprofile.affiliations }}</div>
+        <div class="middle aligned content rel-pl-1">
+          <h1 class="ui medium header" aria-label="{{ _('Your dashboard') }}">{{ current_userprofile.full_name or current_user.username or current_userprofile.email or _('Anonymous user')}}</h1>
+          {% if current_userprofile.affiliations %}
+            <div class="meta">{{ current_userprofile.affiliations }}</div>
+          {% endif %}
         </div>
       </div>
     </div>
+    {% if active_dashboard_menu_item == "uploads" %}
+      <a class="ui tiny button positive left labeled icon m-0" href={{ url_for("invenio_app_rdm_records.deposit_create") }}>
+        <i class="upload icon" aria-hidden="true"></i>
+        {{ _('New upload') }}
+      </a>
+    {% endif %}
+
+    {% if active_dashboard_menu_item == "communities" %}
+      <a class="ui tiny button positive left labeled icon m-0" href={{ url_for("invenio_communities.communities_new") }}>
+        <i class="plus icon" aria-hidden="true"></i>
+        {{ _('New community') }}
+      </a>
+    {% endif %}
   </div>
   <div class="ui container secondary pointing menu page-subheader pl-0">
-  {% for item in current_menu.submenu('dashboard').children %}
+  {% for item in current_menu.submenu('dashboard').children if item.visible %}
   <a
     class="item {{ 'active' if active_dashboard_menu_item == item.name }} {{ 'disabled' if not item.url }}"
     href="{{ item.url }}">{{ item.text }}</a>
   {% endfor %}
 </div>
 </div>
-

invenio_app_rdm/users_ui/views/__init__.py

@@ -11,4 +11,4 @@
 
 from .ui import create_ui_blueprint
 
-__all__ = "create_ui_blueprint"
+__all__ = ("create_ui_blueprint",)