invenio-vocabularies 6.4.1__tar.gz → 6.6.0__tar.gz

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/CHANGES.rst

@@ -8,6 +8,23 @@
 Changes
 =======
 
+Version v6.6.0 (released 2024-11-15)
+
+- mesh: add title en if not present
+- subjects: add subject to search fields
+- jobs: add ORCID job
+- global: Add unlisted tag
+    * This adds a new tag to the vocabularies to allow for unlisted
+      vocabularies. This is useful for vocabularies that are not meant to be
+      displayed in the UI.
+    * This requires to update the names mapping to add the props.
+
+Version v6.5.0 (released 2024-10-31)
+
+- subjects: euroscivoc: change default to latest version-less URL
+- Rename patched filters so the normalizer uses the default ones (#409)
+    * rename patched filters so the normalizer uses the default ones
+
 Version v6.4.1 (released 2024-10-15)
 
 - fix: exclude unknown fields when updating awards with subjects

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 6.4.1
+Version: 6.6.0
 Summary: Invenio module for managing vocabularies.
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,23 @@ Description: ..
         Changes
         =======
         
+        Version v6.6.0 (released 2024-11-15)
+        
+        - mesh: add title en if not present
+        - subjects: add subject to search fields
+        - jobs: add ORCID job
+        - global: Add unlisted tag
+            * This adds a new tag to the vocabularies to allow for unlisted
+              vocabularies. This is useful for vocabularies that are not meant to be
+              displayed in the UI.
+            * This requires to update the names mapping to add the props.
+        
+        Version v6.5.0 (released 2024-10-31)
+        
+        - subjects: euroscivoc: change default to latest version-less URL
+        - Rename patched filters so the normalizer uses the default ones (#409)
+            * rename patched filters so the normalizer uses the default ones
+        
         Version v6.4.1 (released 2024-10-15)
         
         - fix: exclude unknown fields when updating awards with subjects

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "6.4.1"
+__version__ = "6.6.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/config.py

@@ -172,7 +172,7 @@ VOCABULARIES_TYPES_SEARCH = {
 }
 """Vocabulary type search configuration."""
 
-SUBJECTS_EUROSCIVOC_FILE_URL = "https://op.europa.eu/o/opportal-service/euvoc-download-handler?cellarURI=http%3A%2F%2Fpublications.europa.eu%2Fresource%2Fdistribution%2Feuroscivoc%2F20231115-0%2Frdf%2Fskos_ap_eu%2FEuroSciVoc-skos-ap-eu.rdf&fileName=EuroSciVoc-skos-ap-eu.rdf"
+SUBJECTS_EUROSCIVOC_FILE_URL = "https://publications.europa.eu/resource/distribution/euroscivoc/rdf/skos_ap_eu/EuroSciVoc-skos-ap-eu.rdf"
 """Subject EuroSciVoc file download link."""
 
 VOCABULARIES_ORCID_ACCESS_KEY = "TODO"

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -16,7 +16,8 @@ from invenio_records_resources.factories.factory import RecordTypeFactory
 from invenio_records_resources.records.systemfields import ModelPIDField
 from invenio_records_resources.resources.records.headers import etag_headers
 
-from ...services.permissions import PermissionPolicy
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
 from .config import AffiliationsSearchOptions, service_components
 from .schema import AffiliationSchema
 

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/awards/awards.py

@@ -27,7 +27,8 @@ from invenio_records_resources.records.systemfields import (
 )
 from invenio_records_resources.resources.records.headers import etag_headers
 
-from ...services.permissions import PermissionPolicy
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
 from ..funders.api import Funder
 from ..subjects.api import Subject
 from .config import AwardsSearchOptions, service_components

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/funders/funders.py

@@ -21,7 +21,8 @@ from invenio_records_resources.factories.factory import RecordTypeFactory
 from invenio_records_resources.records.systemfields import ModelPIDField
 from invenio_records_resources.resources.records.headers import etag_headers
 
-from ...services.permissions import PermissionPolicy
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
 from .config import FundersSearchOptions, service_components
 from .schema import FunderSchema
 from .serializer import FunderL10NItemSchema

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/funders/mappings/os-v1/funders/funder-v2.0.0.json

@@ -14,8 +14,8 @@
           "type": "custom",
           "char_filter": ["strip_special_chars"],
           "filter": [
-            "lowercase",
-            "asciifolding",
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
             "edgegrams"
           ]
         },
@@ -24,8 +24,8 @@
           "type": "custom",
           "char_filter": ["strip_special_chars"],
           "filter": [
-            "lowercase",
-            "asciifolding"
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
           ]
         }
       },
@@ -40,11 +40,11 @@
         }
       },
       "filter": {
-        "lowercase": {
+        "lowercasepreserveoriginal": {
           "type": "lowercase",
           "preserve_original": true
         },
-        "asciifolding": {
+        "asciifoldingpreserveoriginal": {
           "type": "asciifolding",
           "preserve_original": true
         },

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/funders/mappings/os-v2/funders/funder-v2.0.0.json

@@ -14,8 +14,8 @@
           "type": "custom",
           "char_filter": ["strip_special_chars"],
           "filter": [
-            "lowercase",
-            "asciifolding",
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal",
             "edgegrams"
           ]
         },
@@ -24,8 +24,8 @@
           "type": "custom",
           "char_filter": ["strip_special_chars"],
           "filter": [
-            "lowercase",
-            "asciifolding"
+            "lowercasepreserveoriginal",
+            "asciifoldingpreserveoriginal"
           ]
         }
       },
@@ -40,11 +40,11 @@
         }
       },
       "filter": {
-        "lowercase": {
+        "lowercasepreserveoriginal": {
           "type": "lowercase",
           "preserve_original": true
         },
-        "asciifolding": {
+        "asciifoldingpreserveoriginal": {
           "type": "asciifolding",
           "preserve_original": true
         },

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json

@@ -15,15 +15,38 @@
       "description": "Identifier of the name scheme.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"
     },
-    "name": {"type": "string"},
-    "given_name": {"type": "string"},
-    "family_name": {"type": "string"},
+    "name": {
+      "type": "string"
+    },
+    "given_name": {
+      "type": "string"
+    },
+    "family_name": {
+      "type": "string"
+    },
     "identifiers": {
       "description": "Identifiers for the person.",
       "type": "array",
-      "items": {"$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"},
+      "items": {
+        "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
+      },
       "uniqueItems": true
     },
+    "props": {
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        }
+      }
+    },
     "affiliations": {
       "description": "Affiliations of the person.",
       "type": "array",
@@ -42,4 +65,4 @@
       "uniqueItems": true
     }
   }
-}
+}

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/names/mappings/os-v1/names/name-v2.0.0.json

@@ -144,6 +144,10 @@
             "type": "keyword"
           }
         }
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
       }
     }
   }

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/names/mappings/os-v2/names/name-v2.0.0.json

@@ -144,6 +144,10 @@
             "type": "keyword"
           }
         }
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
       }
     }
   }

invenio-vocabularies-6.6.0/invenio_vocabularies/contrib/names/permissions.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary permissions."""
+
+from invenio_records_permissions.generators import AuthenticatedUser, SystemProcess
+
+from invenio_vocabularies.services.generators import IfTags
+from invenio_vocabularies.services.permissions import PermissionPolicy
+
+
+class NamesPermissionPolicy(PermissionPolicy):
+    """Names permission policy.
+
+    Names endpoints are protected, only authenticated users can access them.
+    """
+
+    can_search = [
+        SystemProcess(),
+        AuthenticatedUser(),
+    ]
+    can_read = [
+        SystemProcess(),
+        IfTags(["unlisted"], then_=[SystemProcess()], else_=[AuthenticatedUser()]),
+    ]

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/names/schema.py

@@ -42,6 +42,7 @@ class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
         )
     )
     affiliations = fields.List(fields.Nested(AffiliationRelationSchema))
+    props = fields.Dict(keys=fields.Str(), values=fields.Raw())
 
     @validates_schema
     def validate_names(self, data, **kwargs):

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/names/services.py

@@ -19,11 +19,12 @@ NamesServiceConfig = record_type.service_config_cls
 class NamesService(record_type.service_cls):
     """Name service."""
 
-    def resolve(self, identity, id_, id_type):
+    def resolve(self, identity, id_, id_type, many=False):
         """Get the record with a given identifier.
 
-        This method assumes that the are no duplicates in the system
-        (i.e. only one name record can have a pair of identifier:scheme).
+        param id_: The identifier value.
+        param id_type: The identifier type.
+        param many: If True, return a list of records.
         """
         search_query = dsl.Q(
             "bool",
@@ -36,20 +37,28 @@ class NamesService(record_type.service_cls):
 
         # max_records = 1, we assume there cannot be duplicates
         # the loading process needs to make sure of that
-        results = self._read_many(identity, search_query, max_records=1)
+        if many:
+            results = self._read_many(identity, search_query)
+        else:
+            results = self._read_many(identity, search_query, max_records=1)
+
         # cant use the results_item because it returns dicts intead of records
         total = results.hits.total["value"]
         if total == 0:
             # Not a PID but trated as such
             raise PIDDoesNotExistError(pid_type=id_type, pid_value=id_)
+        if many:
+            for result in results:
+                record = self.record_cls.loads(result.to_dict())
+                self.require_permission(identity, "read", record=record)
+            return self.result_list(self, identity, results)
+        else:
+            record = self.record_cls.loads(results[0].to_dict())
+            self.require_permission(identity, "read", record=record)
 
-        # (0 < #hits <= max_records) = 1
-        record = self.record_cls.loads(results[0].to_dict())
-        self.require_permission(identity, "read", record=record)
-
-        return self.result_item(
-            self,
-            identity,
-            record,
-            links_tpl=self.links_item_tpl,
-        )
+            return self.result_item(
+                self,
+                identity,
+                record,
+                links_tpl=self.links_item_tpl,
+            )

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/subjects/config.py

@@ -11,6 +11,7 @@
 """Subjects configuration."""
 
 from flask import current_app
+from invenio_i18n import get_locale
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
@@ -22,6 +23,7 @@ from ...services.querystr import FilteredSuggestQueryParser
 subject_schemes = LocalProxy(
     lambda: current_app.config["VOCABULARIES_SUBJECTS_SCHEMES"]
 )
+localized_title = LocalProxy(lambda: f"title.{get_locale()}^20")
 
 
 class SubjectsSearchOptions(SearchOptions):
@@ -30,7 +32,8 @@ class SubjectsSearchOptions(SearchOptions):
     suggest_parser_cls = FilteredSuggestQueryParser.factory(
         filter_field="scheme",
         fields=[  # suggest fields
-            "title.*^100",
+            "subject^100",
+            localized_title,
             "synonyms^20",
         ],
     )

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/contrib/subjects/mesh/datastreams.py

@@ -22,14 +22,19 @@ class MeshSubjectsTransformer(BaseTransformer):
         """Apply transformation on steam entry."""
         entry_data = stream_entry.entry
 
-        # ID in MeSH data is the URL, ex. https://id.nlm.nih.gov/mesh/D000001
+        # ID in MeSH data is in the URL, ex. https://id.nlm.nih.gov/mesh/D000001
         # We just want to use the ID prefixed by "mesh:""
         try:
             mesh_id = entry_data["id"].split("/")[-1]
+            entry_data["id"] = "mesh:" + mesh_id
         except Exception:
             raise TransformerError("Not a valid MeSH ID.")
 
-        entry_data["id"] = "mesh:" + mesh_id
+        entry_data["title"] = title = entry_data.get("title", {})
+        # NOTE: MeSH import file comes with an English subject by default
+        if "en" not in title:
+            title["en"] = entry_data["subject"]
+
         return stream_entry
 
 

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/jobs.py

@@ -18,6 +18,8 @@ from marshmallow_utils.fields import TZDateTime
 
 from invenio_vocabularies.services.tasks import process_datastream
 
+from .contrib.names.datastreams import ORCID_PRESET_DATASTREAM_CONFIG
+
 
 class ArgsSchema(Schema):
     """Schema of task input arguments."""
@@ -194,3 +196,16 @@ class UpdateAwardsCordisJob(ProcessDataStreamJob):
                 ],
             }
         }
+
+
+class ImportORCIDJob(ProcessDataStreamJob):
+    """Import ORCID data registered task."""
+
+    description = "Import ORCID data"
+    title = "Import ORCID data"
+    id = "import_orcid"
+
+    @classmethod
+    def default_args(cls, job_obj, **kwargs):
+        """Generate default job arguments."""
+        return {"config": {**ORCID_PRESET_DATASTREAM_CONFIG}}

invenio-vocabularies-6.6.0/invenio_vocabularies/services/generators.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+#
+
+"""Vocabulary generators."""
+
+from invenio_access import any_user, authenticated_user
+from invenio_records_permissions.generators import ConditionalGenerator
+from invenio_search.engine import dsl
+
+
+class IfTags(ConditionalGenerator):
+    """Generator to filter based on tags.
+
+    This generator will filter out records based on the tags field.
+    """
+
+    def __init__(self, tags, then_, else_):
+        """Constructor."""
+        self.tags = tags or []
+        super().__init__(then_, else_)
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record has the tags."""
+        return any(tag in record.get("tags", []) for tag in self.tags)
+
+    def query_filter(self, **kwargs):
+        """Search based on configured tags."""
+        must_not_clauses = [dsl.Q("terms", tags=self.tags)]
+        return dsl.Q(
+            "bool",
+            must_not=must_not_clauses,
+        )

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies/services/permissions.py

@@ -11,12 +11,17 @@
 from invenio_records_permissions import RecordPermissionPolicy
 from invenio_records_permissions.generators import AnyUser, SystemProcess
 
+from invenio_vocabularies.services.generators import IfTags
+
 
 class PermissionPolicy(RecordPermissionPolicy):
     """Permission policy."""
 
     can_search = [SystemProcess(), AnyUser()]
-    can_read = [SystemProcess(), AnyUser()]
+    can_read = [
+        SystemProcess(),
+        IfTags(["unlisted"], then_=[SystemProcess()], else_=[AnyUser()]),
+    ]
     can_create = [SystemProcess()]
     can_update = [SystemProcess()]
     can_delete = [SystemProcess()]

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 6.4.1
+Version: 6.6.0
 Summary: Invenio module for managing vocabularies.
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,23 @@ Description: ..
         Changes
         =======
         
+        Version v6.6.0 (released 2024-11-15)
+        
+        - mesh: add title en if not present
+        - subjects: add subject to search fields
+        - jobs: add ORCID job
+        - global: Add unlisted tag
+            * This adds a new tag to the vocabularies to allow for unlisted
+              vocabularies. This is useful for vocabularies that are not meant to be
+              displayed in the UI.
+            * This requires to update the names mapping to add the props.
+        
+        Version v6.5.0 (released 2024-10-31)
+        
+        - subjects: euroscivoc: change default to latest version-less URL
+        - Rename patched filters so the normalizer uses the default ones (#409)
+            * rename patched filters so the normalizer uses the default ones
+        
         Version v6.4.1 (released 2024-10-15)
         
         - fix: exclude unknown fields when updating awards with subjects

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies.egg-info/SOURCES.txt

@@ -233,6 +233,7 @@ invenio_vocabularies/services/__init__.py
 invenio_vocabularies/services/components.py
 invenio_vocabularies/services/config.py
 invenio_vocabularies/services/facets.py
+invenio_vocabularies/services/generators.py
 invenio_vocabularies/services/permissions.py
 invenio_vocabularies/services/querystr.py
 invenio_vocabularies/services/results.py
@@ -369,6 +370,7 @@ tests/contrib/funders/test_funders_resource.py
 tests/contrib/funders/test_funders_schema.py
 tests/contrib/funders/test_funders_service.py
 tests/contrib/names/conftest.py
+tests/contrib/names/test_name_permissions.py
 tests/contrib/names/test_names_api.py
 tests/contrib/names/test_names_datastreams.py
 tests/contrib/names/test_names_jsonschema.py

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/invenio_vocabularies.egg-info/entry_points.txt

@@ -51,6 +51,7 @@ invenio_vocabularies = invenio_vocabularies
 
 [invenio_jobs.jobs]
 import_awards_openaire = invenio_vocabularies.jobs:ImportAwardsOpenAIREJob
+import_orcid_names = invenio_vocabularies.jobs:ImportORCIDJob
 process_ror_affiliations = invenio_vocabularies.jobs:ProcessRORAffiliationsJob
 process_ror_funders = invenio_vocabularies.jobs:ProcessRORFundersJob
 update_awards_cordis = invenio_vocabularies.jobs:UpdateAwardsCordisJob

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/setup.cfg

@@ -107,6 +107,7 @@ invenio_jobs.jobs =
 	process_ror_funders = invenio_vocabularies.jobs:ProcessRORFundersJob
 	import_awards_openaire = invenio_vocabularies.jobs:ImportAwardsOpenAIREJob
 	update_awards_cordis = invenio_vocabularies.jobs:UpdateAwardsCordisJob
+	import_orcid_names = invenio_vocabularies.jobs:ImportORCIDJob
 
 [build_sphinx]
 source-dir = docs/

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/tests/conftest.py

@@ -37,7 +37,12 @@ import pytest
 from flask_principal import Identity, Need, UserNeed
 from flask_security import login_user
 from flask_security.utils import hash_password
-from invenio_access.permissions import ActionUsers, any_user, system_process
+from invenio_access.permissions import (
+    ActionUsers,
+    any_user,
+    superuser_access,
+    system_process,
+)
 from invenio_access.proxies import current_access
 from invenio_accounts.proxies import current_datastore
 from invenio_accounts.testutils import login_user_via_session
@@ -113,6 +118,17 @@ def identity():
     return i
 
 
+@pytest.fixture(scope="module")
+def superuser_identity():
+    """Super user identity to interact with the services."""
+    i = Identity(2)
+    i.provides.add(UserNeed(2))
+    i.provides.add(any_user)
+    i.provides.add(system_process)
+    i.provides.add(superuser_access)
+    return i
+
+
 @pytest.fixture(scope="module")
 def service(app):
     """Vocabularies service object."""
@@ -151,6 +167,14 @@ def lang_data2(lang_data):
     return data
 
 
+@pytest.fixture()
+def non_searchable_lang_data(lang_data):
+    """Example data for testing unlisted cases."""
+    data = dict(lang_data)
+    data["tags"] = ["unlisted", "recommended"]
+    return data
+
+
 @pytest.fixture()
 def example_record(db, identity, service, example_data):
     """Example record."""

{invenio-vocabularies-6.4.1 → invenio-vocabularies-6.6.0}/tests/contrib/names/conftest.py

@@ -13,8 +13,11 @@ fixtures are available.
 """
 
 import pytest
+from invenio_indexer.api import RecordIndexer
 from invenio_records_resources.proxies import current_service_registry
 
+from invenio_vocabularies.contrib.names.api import Name
+
 
 @pytest.fixture(scope="module")
 def service():
@@ -56,3 +59,29 @@ def name_full_data():
         ],
         "affiliations": [{"id": "cern"}, {"name": "CustomORG"}],
     }
+
+
+@pytest.fixture(scope="function")
+def non_searchable_name_data():
+    """Full name data."""
+    return {
+        "id": "0000-0001-8135-3489",
+        "name": "Doe, John",
+        "given_name": "John",
+        "family_name": "Doe",
+        "identifiers": [
+            {"identifier": "0000-0001-8135-3489", "scheme": "orcid"},
+            {"identifier": "gnd:4079154-3", "scheme": "gnd"},
+        ],
+        "affiliations": [{"id": "cern"}, {"name": "CustomORG"}],
+        "tags": ["unlisted"],
+    }
+
+
+@pytest.fixture(scope="module")
+def indexer():
+    """Indexer instance with correct Record class."""
+    return RecordIndexer(
+        record_cls=Name,
+        record_to_index=lambda r: (r.__class__.index._name, "_doc"),
+    )