invenio-vocabularies 1.5.1__tar.gz → 2.0.0__tar.gz

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/.github/workflows/tests.yml

@@ -31,28 +31,16 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
         requirements-level: [pypi]
-        db-service: [postgresql10, postgresql13, mysql8]
+        db-service: [postgresql14]
         search-service: [opensearch2, elasticsearch7]
         node-version: [16.x]
-        exclude:
-          - python-version: 3.7
-            db-service: postgresql13
-
-          - python-version: 3.7
-            db-service: mysql8
-
         include:
           - search-service: opensearch2
             SEARCH_EXTRAS: "opensearch2"
-
           - search-service: elasticsearch7
             SEARCH_EXTRAS: "elasticsearch7"
-
-          - db-service: mysql8
-            DB_EXTRAS: "mysql"
-
     env:
       DB: ${{ matrix.db-service }}
       SEARCH: ${{ matrix.search-service }}

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/CHANGES.rst

@@ -8,6 +8,19 @@
 Changes
 =======
 
+Version 2.0.0 (2023-09-14)
+
+- contrib-awards: add "program" to schema fields
+- global: switch names and affiliations to model PID field
+- ci: update matrix
+- awards: add "program" field
+- config: update awards funders mapping
+- service: add sort option to load vocabs
+
+Version 1.6.0 (2023-09-12)
+
+- awards: add acronym to schema
+
 Version 1.5.1 (2023-07-07)
 
 - fix string type columns for mysql

{invenio-vocabularies-1.5.1/invenio_vocabularies.egg-info → invenio-vocabularies-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 1.5.1
+Version: 2.0.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,19 @@ Description: ..
         Changes
         =======
         
+        Version 2.0.0 (2023-09-14)
+        
+        - contrib-awards: add "program" to schema fields
+        - global: switch names and affiliations to model PID field
+        - ci: update matrix
+        - awards: add "program" field
+        - config: update awards funders mapping
+        - service: add sort option to load vocabs
+        
+        Version 1.6.0 (2023-09-12)
+        
+        - awards: add acronym to schema
+        
         Version 1.5.1 (2023-07-07)
         
         - fix string type columns for mysql

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "1.5.1"
+__version__ = "2.0.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

invenio-vocabularies-2.0.0/invenio_vocabularies/alembic/55a700f897b6_add_names_and_afiliations_pid_column.py

@@ -0,0 +1,96 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2023 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Add "pid" column to names and affiliations tables."""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "55a700f897b6"
+down_revision = "e1146238edd3"
+branch_labels = ()
+depends_on = None
+
+
+pids_table = sa.sql.table(
+    "pidstore_pid",
+    sa.sql.column("pid_type"),
+    sa.sql.column("pid_value"),
+    sa.sql.column("object_uuid"),
+    sa.sql.column("object_type"),
+)
+names_table = sa.sql.table(
+    "name_metadata",
+    sa.sql.column("id"),
+    sa.sql.column("pid"),
+)
+affiliations_table = sa.sql.table(
+    "affiliation_metadata",
+    sa.sql.column("id"),
+    sa.sql.column("pid"),
+)
+
+
+def upgrade():
+    """Upgrade database."""
+    op.add_column(
+        "name_metadata",
+        sa.Column("pid", sa.String(255), nullable=True),
+    )
+    # populate from PIDStore
+    op.execute(
+        names_table.update()
+        .where(
+            names_table.c.id == pids_table.c.object_uuid,
+            pids_table.c.object_type == "rec",
+            pids_table.c.pid_type == "names",
+        )
+        .values(pid=pids_table.c.pid_value)
+    )
+    op.create_unique_constraint(
+        op.f("uq_name_metadata_pid"),
+        "name_metadata",
+        ["pid"],
+    )
+
+    op.add_column(
+        "affiliation_metadata",
+        sa.Column("pid", sa.String(255), nullable=True),
+    )
+    # populate from PIDStore
+    op.execute(
+        affiliations_table.update()
+        .where(
+            affiliations_table.c.id == pids_table.c.object_uuid,
+            pids_table.c.object_type == "rec",
+            pids_table.c.pid_type == "aff",
+        )
+        .values(pid=pids_table.c.pid_value)
+    )
+
+    op.create_unique_constraint(
+        op.f("uq_affiliation_metadata_pid"),
+        "affiliation_metadata",
+        ["pid"],
+    )
+
+
+def downgrade():
+    """Downgrade database."""
+    op.drop_constraint(
+        op.f("uq_name_metadata_pid"),
+        "name_metadata",
+        type_="unique",
+    )
+    op.drop_constraint(
+        op.f("uq_affiliation_metadata_pid"),
+        "affiliation_metadata",
+        type_="unique",
+    )
+    op.drop_column("name_metadata", "pid")
+    op.drop_column("affiliation_metadata", "pid")

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/config.py

@@ -62,15 +62,18 @@ VOCABULARIES_AWARD_SCHEMES = {
 """Awards allowed identifier schemes."""
 
 VOCABULARIES_AWARDS_OPENAIRE_FUNDERS = {
-    "anr_________": "00rbzpz17",
     "aka_________": "05k73zm37",
+    "anr_________": "00rbzpz17",
     "arc_________": "05mmh0f86",
+    "asap________": "03zj4c476",
     "cihr________": "01gavpb45",
     "corda_______": "00k4n6c32",
+    "corda_____he": "00k4n6c32",
     "corda__h2020": "00k4n6c32",
     "euenvagency_": "02k4b9v70",
     "fct_________": "00snfqn58",
     "fwf_________": "013tf3c58",
+    "inca________": "03m8vkq32",
     "irb_hr______": "03n51vw80",
     "mestd_______": "01znas443",
     "nhmrc_______": "011kf5r70",
@@ -78,12 +81,12 @@ VOCABULARIES_AWARDS_OPENAIRE_FUNDERS = {
     "nserc_______": "01h531d29",
     "nsf_________": "021nxhr62",
     "nwo_________": "04jsz6e67",
-    "rcuk________": "10.13039/501100000690",
-    "ukri________": "001aqnf71",
+    "rcuk________": "00dq2kk65",  # deprecated funder org
     "sfi_________": "0271asj38",
     "snsf________": "00yjd3n13",
     "sshrc_______": "006cvnv84",
     "tubitakf____": "04w9kkr77",
+    "ukri________": "001aqnf71",
     "wt__________": "029chgv08",
 }
 """Mapping of OpenAIRE and ROR funder codes."""

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -8,12 +8,12 @@
 
 """Vocabulary affiliations."""
 
+from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.records.systemfields import ModelPIDField
 
-from ...records.pidprovider import PIDProviderFactory
-from ...records.systemfields import BaseVocabularyPIDFieldContext
 from ...services.permissions import PermissionPolicy
 from .config import AffiliationsSearchOptions, service_components
 from .schema import AffiliationSchema
@@ -21,17 +21,22 @@ from .schema import AffiliationSchema
 record_type = RecordTypeFactory(
     "Affiliation",
     # Data layer
+    pid_field_cls=ModelPIDField,
     pid_field_kwargs={
-        "create": False,
-        "provider": PIDProviderFactory.create(pid_type="aff"),
-        "context_cls": BaseVocabularyPIDFieldContext,
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
     },
     schema_version="1.0.0",
     schema_path="local://affiliations/affiliation-v1.0.0.json",
     record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
         extensions=[
             IndexedAtDumperExt(),
-        ]
+        ],
     ),
     # Service layer
     service_id="affiliations",

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/affiliations/jsonschemas/affiliations/affiliation-v1.0.0.json

@@ -10,9 +10,6 @@
     "acronym": {
       "type": "string"
     },
-    "id": {
-      "$ref": "local://definitions-v1.0.0.json#/identifier"
-    },
     "identifiers": {
       "description": "Alternate identifiers for the record.",
       "type": "array",
@@ -22,9 +19,6 @@
     "name": {
       "type": "string"
     },
-    "pid": {
-      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
-    },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
     }

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/affiliations/schema.py

@@ -15,11 +15,15 @@ from marshmallow import fields
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
-from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
+from ...services.schema import (
+    BaseVocabularySchema,
+    ContribVocabularyRelationSchema,
+    ModePIDFieldVocabularyMixin,
+)
 from .config import affiliation_schemes
 
 
-class AffiliationSchema(BaseVocabularySchema):
+class AffiliationSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
     """Service schema for affiliations."""
 
     acronym = SanitizedUnicode()

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/awards/datastreams.py

@@ -52,6 +52,15 @@ class OpenAIREProjectTransformer(BaseTransformer):
 
         award["id"] = f"{funder_id}::{code}"
 
+        funding = next(iter(record.get("funding", [])), None)
+        if funding:
+            funding_stream_id = funding.get("funding_stream", {}).get("id", "")
+            # Example funding stream ID: `EC::HE::HORIZON-AG-UN`. We need the `EC`
+            # string, i.e. the second "part" of the identifier.
+            program = next(iter(funding_stream_id.split("::")[1:2]), "")
+            if program:
+                award["program"] = program
+
         identifiers = []
         if funder_id == awards_ec_ror_id:
             identifiers.append(

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/awards/jsonschemas/awards/award-v1.0.0.json

@@ -36,6 +36,9 @@
     },
     "acronym": {
       "type": "string"
+    },
+    "program": {
+      "type": "string"
     }
   }
 }

{invenio-vocabularies-1.5.1/invenio_vocabularies/contrib/awards/mappings/os-v2 → invenio-vocabularies-2.0.0/invenio_vocabularies/contrib/awards/mappings/os-v1}/awards/award-v1.0.0.json

@@ -55,6 +55,9 @@
       "acronym": {
         "type": "keyword"
       },
+      "program": {
+        "type": "keyword"
+      },
       "funder": {
         "type": "object",
         "properties": {

{invenio-vocabularies-1.5.1/invenio_vocabularies/contrib/awards/mappings/v7 → invenio-vocabularies-2.0.0/invenio_vocabularies/contrib/awards/mappings/os-v2}/awards/award-v1.0.0.json

@@ -55,6 +55,9 @@
       "acronym": {
         "type": "keyword"
       },
+      "program": {
+        "type": "keyword"
+      },
       "funder": {
         "type": "object",
         "properties": {

{invenio-vocabularies-1.5.1/invenio_vocabularies/contrib/awards/mappings/os-v1 → invenio-vocabularies-2.0.0/invenio_vocabularies/contrib/awards/mappings/v7}/awards/award-v1.0.0.json

@@ -55,6 +55,9 @@
       "acronym": {
         "type": "keyword"
       },
+      "program": {
+        "type": "keyword"
+      },
       "funder": {
         "type": "object",
         "properties": {

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/awards/schema.py

@@ -10,26 +10,21 @@
 
 from functools import partial
 
-from attr import attr
 from invenio_i18n import lazy_gettext as _
-from marshmallow import (
-    Schema,
-    ValidationError,
-    fields,
-    post_load,
-    pre_dump,
-    validate,
-    validates_schema,
-)
+from marshmallow import Schema, ValidationError, fields, validate, validates_schema
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
-from ...services.schema import BaseVocabularySchema, i18n_strings
+from ...services.schema import (
+    BaseVocabularySchema,
+    ModePIDFieldVocabularyMixin,
+    i18n_strings,
+)
 from ..funders.schema import FunderRelationSchema
 from .config import award_schemes
 
 
-class AwardSchema(BaseVocabularySchema):
+class AwardSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
     """Award schema."""
 
     identifiers = IdentifierSet(
@@ -49,32 +44,12 @@ class AwardSchema(BaseVocabularySchema):
 
     acronym = SanitizedUnicode()
 
+    program = SanitizedUnicode()
+
     id = SanitizedUnicode(
         validate=validate.Length(min=1, error=_("PID cannot be blank."))
     )
 
-    @validates_schema
-    def validate_id(self, data, **kwargs):
-        """Validates ID."""
-        is_create = "record" not in self.context
-        if is_create and "id" not in data:
-            raise ValidationError(_("Missing PID."), "id")
-        if not is_create:
-            data.pop("id", None)
-
-    @post_load(pass_many=False)
-    def move_id(self, data, **kwargs):
-        """Moves id to pid."""
-        if "id" in data:
-            data["pid"] = data.pop("id")
-        return data
-
-    @pre_dump(pass_many=False)
-    def extract_pid_value(self, data, **kwargs):
-        """Extracts the PID value."""
-        data["id"] = data.pid.pid_value
-        return data
-
 
 class AwardRelationSchema(Schema):
     """Award relation schema."""
@@ -91,6 +66,8 @@ class AwardRelationSchema(Schema):
             )
         )
     )
+    acronym = SanitizedUnicode()
+    program = SanitizedUnicode()
 
     @validates_schema
     def validate_data(self, data, **kwargs):

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/awards/serializer.py

@@ -35,5 +35,6 @@ class AwardL10NItemSchema(Schema):
     description = L10NString(data_key="description_l10n")
     number = fields.String(dump_only=True)
     acronym = fields.String(dump_only=True)
+    program = fields.String(dump_only=True)
     funder = fields.Nested(FunderRelationSchema, dump_only=True)
     identifiers = fields.List(fields.Nested(IdentifierSchema), dump_only=True)

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/names/datastreams.py

@@ -9,12 +9,9 @@
 """Names datastreams, transformers, writers and readers."""
 
 from invenio_access.permissions import system_identity
-from invenio_pidstore.errors import PIDDoesNotExistError
 from invenio_records.dictutils import dict_lookup
-from marshmallow import ValidationError
 
-from ...datastreams import StreamEntry
-from ...datastreams.errors import TransformerError, WriterError
+from ...datastreams.errors import TransformerError
 from ...datastreams.readers import SimpleHTTPReader
 from ...datastreams.transformers import BaseTransformer
 from ...datastreams.writers import ServiceWriter
@@ -40,13 +37,14 @@ class OrcidTransformer(BaseTransformer):
         """Applies the transformation to the stream entry."""
         record = stream_entry.entry
         person = record["person"]
-        orcid_id = record["orcid-identifier"]["uri"]
+        orcid_id = record["orcid-identifier"]["path"]
 
         name = person.get("name")
         if name is None:
             raise TransformerError(f"Name not found in ORCiD entry.")
 
         entry = {
+            "id": orcid_id,
             "given_name": name.get("given-names"),
             "family_name": name.get("family-name"),
             "identifiers": [{"scheme": "orcid", "identifier": orcid_id}],
@@ -79,41 +77,14 @@ class OrcidTransformer(BaseTransformer):
 class NamesServiceWriter(ServiceWriter):
     """Names service writer."""
 
-    def __init__(self, *args, scheme_id="orcid", **kwargs):
+    def __init__(self, *args, **kwargs):
         """Constructor."""
         service_or_name = kwargs.pop("service_or_name", "names")
         super().__init__(service_or_name=service_or_name, *args, **kwargs)
-        self._scheme_id = scheme_id
 
     def _entry_id(self, entry):
         """Get the id from an entry."""
-        for identifier in entry.get("identifiers"):
-            if identifier.get("scheme") == self._scheme_id:
-                return identifier["identifier"]
-
-    def _resolve(self, id_):
-        """Resolve an entry given an id."""
-        return self._service.resolve(self._identity, id_=id_, id_type=self._scheme_id)
-
-    def write(self, stream_entry, *args, **kwargs):
-        """Writes the input entry using a given service."""
-        entry = stream_entry.entry
-        try:
-            vocab_id = self._entry_id(entry)
-            # it is resolved before creation to avoid duplicates since
-            # the pid is recidv2 not e.g. the orcid
-            current = self._resolve(vocab_id)
-            if not self._update:
-                raise WriterError([f"Vocabulary entry already exists: {entry}"])
-            updated = dict(current.to_dict(), **entry)
-            return StreamEntry(
-                self._service.update(self._identity, current.id, updated)
-            )
-        except PIDDoesNotExistError:
-            return StreamEntry(self._service.create(self._identity, entry))
-
-        except ValidationError as err:
-            raise WriterError([{"ValidationError": err.messages}])
+        return entry["id"]
 
 
 VOCABULARIES_DATASTREAM_READERS = {

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json

@@ -8,13 +8,6 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
-    "id": {
-      "description": "URI or classification code as identifier - globally unique among all names schemes.",
-      "$ref": "local://definitions-v1.0.0.json#/identifier"
-    },
-    "pid": {
-      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
-    },
     "scheme": {
       "description": "Identifier of the name scheme.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/names/names.py

@@ -8,16 +8,17 @@
 
 """Vocabulary names."""
 
-from invenio_pidstore.providers.recordid_v2 import RecordIdProviderV2
+from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records.dumpers.relations import RelationDumperExt
 from invenio_records.systemfields import RelationsField
 from invenio_records_resources.factories.factory import RecordTypeFactory
-from invenio_records_resources.records.systemfields import PIDListRelation
+from invenio_records_resources.records.systemfields import (
+    ModelPIDField,
+    PIDListRelation,
+)
 
-from ...records.pidprovider import PIDProviderFactory
-from ...records.systemfields import BaseVocabularyPIDFieldContext
 from ...services.permissions import PermissionPolicy
 from ..affiliations.api import Affiliation
 from .config import NamesSearchOptions, service_components
@@ -35,21 +36,24 @@ name_relations = RelationsField(
 record_type = RecordTypeFactory(
     "Name",
     # Data layer
+    pid_field_cls=ModelPIDField,
     pid_field_kwargs={
-        "create": False,
-        "provider": PIDProviderFactory.create(
-            pid_type="names", base_cls=RecordIdProviderV2
-        ),
-        "context_cls": BaseVocabularyPIDFieldContext,
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
     },
     schema_version="1.0.0",
     schema_path="local://names/name-v1.0.0.json",
     record_relations=name_relations,
     record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
         extensions=[
             RelationDumperExt("relations"),
             IndexedAtDumperExt(),
-        ]
+        ],
     ),
     # Service layer
     service_id="names",

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/contrib/names/schema.py

@@ -11,16 +11,16 @@
 from functools import partial
 
 from invenio_i18n import lazy_gettext as _
-from invenio_records_resources.services.records.schema import BaseRecordSchema
 from marshmallow import ValidationError, fields, post_load, validates_schema
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
+from ...services.schema import BaseVocabularySchema, ModePIDFieldVocabularyMixin
 from ..affiliations.schema import AffiliationRelationSchema
 from .config import names_schemes
 
 
-class NameSchema(BaseRecordSchema):
+class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
     """Service schema for names.
 
     Note that this vocabulary is very different from the base vocabulary

{invenio-vocabularies-1.5.1 → invenio-vocabularies-2.0.0}/invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -30,7 +30,13 @@ class VocabularyCF(BaseCF):
     """
 
     def __init__(
-        self, name, vocabulary_id, multiple=False, dump_options=True, **kwargs
+        self,
+        name,
+        vocabulary_id,
+        multiple=False,
+        dump_options=True,
+        sort_by=None,
+        **kwargs
     ):
         """Constructor."""
         super().__init__(name, **kwargs)
@@ -38,6 +44,7 @@ class VocabularyCF(BaseCF):
         self.vocabulary_id = vocabulary_id
         self.dump_options = dump_options
         self.multiple = multiple
+        self.sort_by = sort_by
 
     @property
     def mapping(self):
@@ -78,6 +85,7 @@ class VocabularyCF(BaseCF):
                 identity,
                 fields=self.field_keys,
                 type=self.vocabulary_id,
+                sort=self.sort_by,
             )
             options = []
             for vocab in vocabs: