invenio-vocabularies 2.3.1__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/names/s3client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""S3 client."""
+
+from flask import current_app
+
+try:
+    import s3fs
+except ImportError:
+    s3fs = None
+
+
+class S3Client:
+    """S3 client."""
+
+    def __init__(self, access_key, secret_key):
+        """Constructor."""
+        if s3fs is None:
+            raise Exception("s3fs is not installed.")
+
+        self.fs = s3fs.S3FileSystem(key=access_key, secret=secret_key)
+
+    def read_file(self, s3_path):
+        """Reads a file from S3."""
+        with self.fs.open(s3_path, "rb") as f:
+            return f.read()
+
+
+class S3OrcidClient(S3Client):
+    """S3 ORCiD client."""
+
+    def __init__(self):
+        """Constructor."""
+        super().__init__(
+            access_key=current_app.config["VOCABULARIES_ORCID_ACCESS_KEY"],
+            secret_key=current_app.config["VOCABULARIES_ORCID_SECRET_KEY"],
+        )

invenio_vocabularies/contrib/names/schema.py

@@ -56,6 +56,20 @@ class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
             ]
             raise ValidationError({"family_name": messages})
 
+    @validates_schema
+    def validate_affiliatons(self, data, **kwargs):
+        """Validate names."""
+        affiliations = data.get("affiliations", [])
+        seen_names = set()
+        for affiliation in affiliations:
+            name = affiliation.get("name")
+            if not affiliation.get("id") and name:
+                if name in seen_names:
+                    messages = [_("Duplicated affiliations.")]
+                    raise ValidationError({"affiliations": messages})
+                else:
+                    seen_names.add(name)
+
     @post_load
     def update_name(self, data, **kwargs):
         """Update names for person.

invenio_vocabularies/contrib/subjects/config.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2021 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,13 +10,19 @@
 
 """Subjects configuration."""
 
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
+from werkzeug.local import LocalProxy
 
 from ...services.components import PIDComponent
 from ...services.querystr import FilteredSuggestQueryParser
 
+subject_schemes = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_SUBJECTS_SCHEMES"]
+)
+
 
 class SubjectsSearchOptions(SearchOptions):
     """Search options."""
@@ -23,9 +30,8 @@ class SubjectsSearchOptions(SearchOptions):
     suggest_parser_cls = FilteredSuggestQueryParser.factory(
         filter_field="scheme",
         fields=[  # suggest fields
-            "subject^100",
-            "subject._2gram",
-            "subject._3gram",
+            "title.*^100",
+            "synonyms^20",
         ],
     )
 

invenio_vocabularies/contrib/subjects/datastreams.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 University of Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Names datastreams, transformers, writers and readers."""
+
+from invenio_access.permissions import system_identity
+from invenio_i18n import lazy_gettext as _
+
+from ...datastreams.writers import ServiceWriter
+from .euroscivoc import datastreams as euroscivoc_datastreams
+from .mesh import datastreams as mesh_datastreams
+
+
+class SubjectsServiceWriter(ServiceWriter):
+    """Subjects Service Writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "subjects")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_READERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_READERS,
+}
+"""Subjects Data Streams readers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_TRANSFORMERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_TRANSFORMERS,
+}
+"""Subjects Data Streams transformers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "subjects-service": SubjectsServiceWriter,
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_WRITERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_WRITERS,
+}
+"""Subjects Data Streams writers."""
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {"type": "yaml"},
+    ],
+    "writers": [
+        {
+            "type": "subjects-service",
+        }
+    ],
+}
+"""Data Stream configuration."""

invenio_vocabularies/contrib/subjects/euroscivoc/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""EuroSciVoc Subjects module."""

invenio_vocabularies/contrib/subjects/euroscivoc/datastreams.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""EuroSciVoc subjects datastreams, readers, transformers, and writers."""
+
+import io
+from collections import namedtuple
+
+import requests
+from rdflib import OWL, RDF, Graph, Namespace
+
+from invenio_vocabularies.config import SUBJECTS_EUROSCIVOC_FILE_URL
+from invenio_vocabularies.datastreams.readers import BaseReader
+from invenio_vocabularies.datastreams.transformers import BaseTransformer
+
+
+class EuroSciVocSubjectsHTTPReader(BaseReader):
+    """Reader class to fetch and process EuroSciVoc RDF data."""
+
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Initialize the reader with the data source.
+
+        :param origin: The URL from which to fetch the RDF data.
+        :param mode: Mode of operation (default is 'r' for reading).
+        """
+        self.origin = origin or SUBJECTS_EUROSCIVOC_FILE_URL
+        super().__init__(origin=origin, mode=mode, *args, **kwargs)
+
+    def _iter(self, rdf_graph):
+        """Iterate over the RDF graph, yielding one subject at a time.
+
+        :param rdf_graph: The RDF graph to process.
+        :yield: Subject and graph to be transformed.
+        """
+        SKOS_CORE = Namespace("http://www.w3.org/2004/02/skos/core#")
+
+        for subject, _, _ in rdf_graph.triples((None, RDF.type, SKOS_CORE.Concept)):
+            yield {"subject": subject, "rdf_graph": rdf_graph}
+
+    def read(self, item=None, *args, **kwargs):
+        """Fetch and process the EuroSciVoc RDF data, yielding it one subject at a time.
+
+        :param item: The RDF data provided as bytes (optional).
+        :yield: Processed EuroSciVoc subject data.
+        """
+        if item:
+            raise NotImplementedError(
+                "EuroSciVocSubjectsHTTPReader does not support being chained after another reader"
+            )
+        # Fetch the RDF data from the specified origin URL
+        response = requests.get(self.origin)
+        response.raise_for_status()
+
+        # Treat the response content as a file-like object
+        rdf_data = io.BytesIO(response.content)
+
+        # Parse the RDF data into a graph
+        rdf_graph = Graph()
+        rdf_graph.parse(rdf_data, format="xml")
+
+        # Yield each processed subject from the RDF graph
+        yield from self._iter(rdf_graph)
+
+
+class EuroSciVocSubjectsTransformer(BaseTransformer):
+    """Transformer class to convert EuroSciVoc RDF data to a dictionary format."""
+
+    SKOS_CORE = Namespace("http://www.w3.org/2004/02/skos/core#")
+    SPLITCHAR = ","
+
+    def _get_notation(self, subject, rdf_graph):
+        """Extract the numeric notation for a subject."""
+        for _, _, notation in rdf_graph.triples(
+            (subject, self.SKOS_CORE.notation, None)
+        ):
+            if str(notation).isdigit():
+                return str(notation)
+        return None
+
+    def _get_labels(self, subject, rdf_graph):
+        """Extract prefLabel and altLabel languages for a subject."""
+        labels = {
+            label.language: label.value.capitalize()
+            for _, _, label in rdf_graph.triples(
+                (subject, self.SKOS_CORE.prefLabel, None)
+            )
+        }
+        if "en" not in labels:
+            for _, _, label in rdf_graph.triples(
+                (subject, self.SKOS_CORE.altLabel, None)
+            ):
+                labels.setdefault(label.language, label.value.capitalize())
+        return labels
+
+    def _find_parents(self, subject, rdf_graph):
+        """Find parent notations."""
+        parents = []
+
+        # Traverse the broader hierarchy
+        for broader in rdf_graph.transitive_objects(subject, self.SKOS_CORE.broader):
+            if broader != subject:  # Ensure we don't include the current subject
+                parent_notation = self._get_notation(broader, rdf_graph)
+                if parent_notation:
+                    parents.append(parent_notation)
+
+        return parents
+
+    def _transform_entry(self, subject, rdf_graph):
+        """Transform an entry to the required dictionary format."""
+        # Get subject notation with euroscivoc prefix
+        notation = self._get_notation(subject, rdf_graph)
+        id = f"euroscivoc:{notation}" if notation else None
+        # Get labels for the current subject
+        labels = self._get_labels(subject, rdf_graph)
+        # Join parent notations with SPLITCHAR separator and add euroscivoc prefix
+        parents = self.SPLITCHAR.join(
+            f"euroscivoc:{n}" for n in reversed(self._find_parents(subject, rdf_graph))
+        )
+        # Create identifiers list
+        identifiers = [{"scheme": "url", "identifier": str(subject)}]
+
+        return {
+            "id": id,
+            "scheme": "EuroSciVoc",
+            "subject": labels.get("en", "").capitalize(),
+            "title": labels,
+            "props": {"parents": parents} if parents else {},
+            "identifiers": identifiers,
+        }
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Transform a stream entry to the required dictionary format.
+
+        :param stream_entry: The entry to be transformed, which includes the subject and the RDF graph.
+        :return: The transformed stream entry.
+        """
+        # Apply transformations
+        entry_data = self._transform_entry(
+            stream_entry.entry["subject"], stream_entry.entry["rdf_graph"]
+        )
+        stream_entry.entry = entry_data
+        return stream_entry
+
+
+# Configuration for datastream readers, transformers, and writers
+VOCABULARIES_DATASTREAM_READERS = {"euroscivoc-reader": EuroSciVocSubjectsHTTPReader}
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "euroscivoc-transformer": EuroSciVocSubjectsTransformer
+}
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "euroscivoc-reader",
+        }
+    ],
+    "transformers": [{"type": "euroscivoc-transformer"}],
+    "writers": [
+        {
+            "type": "subjects-service",
+        }
+    ],
+}

invenio_vocabularies/contrib/subjects/jsonschemas/subjects/subject-v1.0.0.json

@@ -8,6 +8,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "id": {
       "description": "URI or classification code as identifier - globally unique among all subject schemes.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"
@@ -22,6 +25,34 @@
     "subject": {
       "description": "Human readable label.",
       "type": "string"
+    },
+    "title": {
+      "description": "Human readable label in different languages.",
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "props": {
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "type": "string"
+        }
+      }
+    },
+    "identifiers": {
+      "description": "Alternate identifiers for the subject.",
+      "type": "array",
+      "items": {
+        "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "synonyms": {
+      "description": "Synonyms of the subject label.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
     }
   }
 }

invenio_vocabularies/contrib/subjects/mappings/os-v1/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/os-v2/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/v7/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mesh/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH Subjects module."""

invenio_vocabularies/contrib/subjects/mesh/datastreams.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH subjects datastreams, transformers, writers and readers."""
+
+from invenio_vocabularies.datastreams.transformers import (
+    BaseTransformer,
+    TransformerError,
+)
+
+
+class MeshSubjectsTransformer(BaseTransformer):
+    """MeSH subjects Transformer."""
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Apply transformation on steam entry."""
+        entry_data = stream_entry.entry
+
+        # ID in MeSH data is the URL, ex. https://id.nlm.nih.gov/mesh/D000001
+        # We just want to use the ID prefixed by "mesh:""
+        try:
+            mesh_id = entry_data["id"].split("/")[-1]
+        except Exception:
+            raise TransformerError("Not a valid MeSH ID.")
+
+        entry_data["id"] = "mesh:" + mesh_id
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""MeSH datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+"""MeSH subject datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"mesh-subjects": MeshSubjectsTransformer}
+"""MeSH subjects datastream transformers."""

invenio_vocabularies/contrib/subjects/schema.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021 Northwestern University.
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,10 +10,19 @@
 
 """Subjects schema."""
 
-from invenio_i18n import lazy_gettext as _
-from marshmallow_utils.fields import SanitizedUnicode
+from functools import partial
 
-from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
+from invenio_i18n import get_locale
+from marshmallow import Schema, fields, pre_load
+from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
+from marshmallow_utils.schemas import IdentifierSchema
+
+from ...services.schema import (
+    BaseVocabularySchema,
+    ContribVocabularyRelationSchema,
+    i18n_strings,
+)
+from .config import subject_schemes
 
 
 class SubjectSchema(BaseVocabularySchema):
@@ -24,6 +34,26 @@ class SubjectSchema(BaseVocabularySchema):
     id = SanitizedUnicode(required=True)
     scheme = SanitizedUnicode(required=True)
     subject = SanitizedUnicode(required=True)
+    title = i18n_strings
+    props = fields.Dict(keys=SanitizedUnicode(), values=SanitizedUnicode())
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode())
+
+    @pre_load
+    def add_subject_from_title(self, data, **kwargs):
+        """Add subject from title if not present."""
+        locale = get_locale().language
+        if "subject" not in data:
+            data["subject"] = data["title"].get(locale) or data["title"].values()[0]
+        return data
 
 
 class SubjectRelationSchema(ContribVocabularyRelationSchema):
@@ -32,4 +62,16 @@ class SubjectRelationSchema(ContribVocabularyRelationSchema):
     ftf_name = "subject"
     parent_field_name = "subjects"
     subject = SanitizedUnicode()
-    scheme = SanitizedUnicode()
+    scheme = SanitizedUnicode(dump_only=True)
+    title = fields.Dict(dump_only=True)
+    props = fields.Dict(dump_only=True)
+    identifiers = IdentifierSet(
+        fields.Nested(
+            partial(
+                IdentifierSchema,
+                allowed_schemes=subject_schemes,
+                identifier_required=False,
+            )
+        )
+    )
+    synonyms = fields.List(SanitizedUnicode(), dump_only=True)