invenio-vocabularies 4.4.0__tar.gz → 5.0.0__tar.gz

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/.github/workflows/tests.yml

@@ -13,10 +13,13 @@ on:
   push:
     branches:
       - master
+      - "feature/*"
+      - "maint-**"
   pull_request:
     branches:
       - master
       - "maint-**"
+      - "feature/*"
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 3 * * 6"

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/CHANGES.rst

@@ -8,6 +8,10 @@
 Changes
 =======
 
+Version v5.0.0 (released 2024-08-22)
+
+- affiliations: dd analyzers and filters to improve results when searching affiliations
+
 Version v4.4.0 (released 2024-08-09)
 
 - services: use and adjust vnd.inveniordm.v1+json http accept header

{invenio-vocabularies-4.4.0/invenio_vocabularies.egg-info → invenio-vocabularies-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: invenio-vocabularies
-Version: 4.4.0
+Version: 5.0.0
 Summary: "Invenio module for managing vocabularies."
 Home-page: https://github.com/inveniosoftware/invenio-vocabularies
 Author: CERN
@@ -50,6 +50,10 @@ Description: ..
         Changes
         =======
         
+        Version v5.0.0 (released 2024-08-22)
+        
+        - affiliations: dd analyzers and filters to improve results when searching affiliations
+        
         Version v4.4.0 (released 2024-08-09)
         
         - services: use and adjust vnd.inveniordm.v1+json http accept header

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/__init__.py

@@ -10,6 +10,6 @@
 
 from .ext import InvenioVocabularies
 
-__version__ = "4.4.0"
+__version__ = "5.0.0"
 
 __all__ = ("__version__", "InvenioVocabularies")

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/package.json

@@ -13,7 +13,6 @@
     "coveralls": "^3.0.0",
     "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.15.0",
-    "enzyme-to-json": "^3.4.0",
     "expect": "^26.0.0",
     "lodash": "^4.17.0",
     "luxon": "^1.23.0",
@@ -22,10 +21,5 @@
     "react-scripts": "^5.0.1",
     "semantic-ui-react": "^2.1.0",
     "react-overridable": "^0.0.3"
-  },
-  "jest": {
-    "snapshotSerializers": [
-      "enzyme-to-json/serializer"
-    ]
   }
 }

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/config.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -104,7 +105,11 @@ VOCABULARIES_NAMES_SCHEMES = {
 }
 """Names allowed identifier schemes."""
 
-# configure CUSTOM_VOCABULARY_TYPES to differentiate output. Is used in VocabulariesServiceConfig
+VOCABULARIES_SUBJECTS_SCHEMES = {
+    "gnd": {"label": _("GND"), "validator": idutils.is_gnd, "datacite": "GND"},
+}
+"""Subjects allowed identifier schemes."""
+
 VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
     "names",
     "affiliations",
@@ -112,6 +117,7 @@ VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
     "funders",
     "subjects",
 ]
+"""List of custom vocabulary types."""
 
 VOCABULARIES_DATASTREAM_READERS = {
     "csv": CSVReader,

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -34,6 +34,7 @@ record_type = RecordTypeFactory(
     },
     schema_version="1.0.0",
     schema_path="local://affiliations/affiliation-v1.0.0.json",
+    index_name="affiliations-affiliation-v2.0.0",
     record_dumper=SearchDumper(
         model_fields={"pid": ("id", str)},
         extensions=[

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/contrib/affiliations/config.py

@@ -9,6 +9,7 @@
 """Vocabulary affiliations configuration."""
 
 from flask import current_app
+from invenio_i18n import get_locale
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
@@ -20,13 +21,23 @@ from ...services.components import PIDComponent
 affiliation_schemes = LocalProxy(
     lambda: current_app.config["VOCABULARIES_AFFILIATION_SCHEMES"]
 )
+localized_title = LocalProxy(lambda: f"title.{get_locale()}^20")
 
 
 class AffiliationsSearchOptions(SearchOptions):
     """Search options."""
 
     suggest_parser_cls = SuggestQueryParser.factory(
-        fields=["name^100", "acronym^50", "title.*^20", "id^20", "aliases^20"],
+        fields=[
+            "name^100",
+            "acronym.keyword^100",
+            "acronym^40",
+            localized_title,
+            "id^20",
+            "aliases^20",
+        ],
+        type="most_fields",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#multi-match-types
+        fuzziness="AUTO",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness
     )
 
     sort_default = "bestmatch"

invenio-vocabularies-5.0.0/invenio_vocabularies/contrib/affiliations/mappings/os-v1/affiliations/affiliation-v2.0.0.json

@@ -0,0 +1,171 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio-vocabularies-5.0.0/invenio_vocabularies/contrib/affiliations/mappings/os-v2/affiliations/affiliation-v2.0.0.json

@@ -0,0 +1,171 @@
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "strip_special_chars": {
+          "type": "pattern_replace",
+          "pattern": "[\\p{Punct}\\p{S}]",
+          "replacement": ""
+        }
+      },
+      "analyzer": {
+        "accent_edge_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "edgegrams"
+          ]
+        },
+        "accent_analyzer": {
+          "tokenizer": "standard",
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "normalizer": {
+        "accent_normalizer": {
+          "type": "custom",
+          "char_filter": ["strip_special_chars"],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      },
+      "filter": {
+        "lowercase": {
+          "type": "lowercase",
+          "preserve_original": true
+        },
+        "asciifolding": {
+          "type": "asciifolding",
+          "preserve_original": true
+        },
+        "edgegrams": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 20
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "text",
+            "analyzer": "accent_edge_analyzer",
+            "search_analyzer": "accent_analyzer"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "name_sort": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "copy_to": "name_sort"
+      },
+      "acronym": {
+        "type": "text",
+        "analyzer": "accent_edge_analyzer",
+        "search_analyzer": "accent_analyzer",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "normalizer": "accent_normalizer"
+          }
+        }
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "country": {
+        "type": "text"
+      },
+      "country_name": {
+        "type": "text"
+      },
+      "location_name": {
+        "type": "text"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "aliases": {
+        "type": "text"
+      },
+      "types": {
+        "type": "keyword"
+      }
+    }
+  }
+}

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/contrib/common/ror/datastreams.py

@@ -10,8 +10,8 @@
 """ROR-related Datastreams Readers/Writers/Transformers module."""
 
 import io
-from datetime import datetime
 
+import arrow
 import requests
 from idutils import normalize_ror
 
@@ -33,6 +33,26 @@ class RORHTTPReader(BaseReader):
             "RORHTTPReader downloads one file and therefore does not iterate through items"
         )
 
+    def _get_last_dump_date(self, linksets):
+        """Get the last dump date."""
+        for linkset in linksets:
+            metadata_formats = linkset.get("describedby", [])
+            for format_link in metadata_formats:
+                if format_link.get("type") == "application/ld+json":
+                    json_ld_reponse = requests.get(
+                        format_link["href"],
+                        headers={"Accept": format_link["type"]},
+                    )
+                    json_ld_reponse.raise_for_status()
+                    json_ld_data = json_ld_reponse.json()
+
+                    last_dump_date = arrow.get(json_ld_data["dateCreated"])
+                    return last_dump_date
+        else:
+            raise ReaderError(
+                "Couldn't find JSON-LD in publisher's linkset to determine last dump date."
+            )
+
     def read(self, item=None, *args, **kwargs):
         """Reads the latest ROR data dump ZIP file from Zenodo and yields an in-memory binary stream of it."""
         if item:
@@ -54,39 +74,21 @@ class RORHTTPReader(BaseReader):
             headers={"Accept": "application/linkset+json"},
         )
         linkset_response.raise_for_status()
+        linksets = linkset_response.json()["linkset"]
 
         if self._since:
-            for link in linkset_response.json()["linkset"]:
-                if "type" in link and link["type"] == "application/ld+json":
-                    json_ld_reponse = requests.get(
-                        link["anchor"], headers={"Accept": link["type"]}
-                    )
-                    json_ld_reponse.raise_for_status()
-
-                    # TODO Update to use dateCreated once the field is added to InvenioRDM. (https://github.com/inveniosoftware/invenio-rdm-records/issues/1777)
-                    last_dump_date = json_ld_reponse.json()["datePublished"]
-                    if datetime.fromisoformat(last_dump_date) < datetime.fromisoformat(
-                        self._since
-                    ):
-                        return
-                    break
-            else:
-                raise ReaderError("Couldn't find json-ld in publisher's linkset.")
-
-        # Extract the Landing page Link Set Object located as the first (index 0) item.
-        landing_page_linkset = linkset_response.json()["linkset"][0]
-
-        # Extract the URL of the only ZIP file linked to the record.
-        landing_page_zip_items = [
-            item
-            for item in landing_page_linkset["item"]
-            if item["type"] == "application/zip"
-        ]
-        if len(landing_page_zip_items) != 1:
-            raise ReaderError(
-                f"Expected 1 ZIP item but got {len(landing_page_zip_items)}"
-            )
-        file_url = landing_page_zip_items[0]["href"]
+            last_dump_date = self._get_last_dump_date(linksets)
+            if last_dump_date < arrow.get(self._since):
+                return
+
+        for linkset in linksets:
+            items = linkset.get("item", [])
+            zip_files = [item for item in items if item["type"] == "application/zip"]
+            if len(zip_files) == 1:
+                file_url = zip_files[0]["href"]
+                break
+            if len(zip_files) > 1:
+                raise ReaderError(f"Expected 1 ZIP item but got {len(zip_files)}")
 
         # Download the ZIP file and fully load the response bytes content in memory.
         # The bytes content are then wrapped by a BytesIO to be file-like object (as required by `zipfile.ZipFile`).

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/contrib/funders/config.py

@@ -33,7 +33,9 @@ class FundersSearchOptions(SearchOptions):
             "identifiers.identifier^10",
             "acronym^10",
             "aliases^10",
-        ]
+        ],
+        type="most_fields",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#multi-match-types
+        fuzziness="AUTO",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness
     )
 
     sort_default = "bestmatch"

{invenio-vocabularies-4.4.0 → invenio-vocabularies-5.0.0}/invenio_vocabularies/contrib/funders/funders.py

@@ -46,6 +46,7 @@ record_type = RecordTypeFactory(
     ),
     schema_version="1.0.0",
     schema_path="local://funders/funder-v1.0.0.json",
+    index_name="funders-funder-v2.0.0",
     # Service layer
     service_id="funders",
     service_schema=FunderSchema,