invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/contrib/names/names.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,17 +8,21 @@
 
 """Vocabulary names."""
 
-from invenio_pidstore.providers.recordid_v2 import RecordIdProviderV2
+from flask_resources import JSONSerializer, ResponseHandler
+from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records.dumpers.relations import RelationDumperExt
 from invenio_records.systemfields import RelationsField
 from invenio_records_resources.factories.factory import RecordTypeFactory
-from invenio_records_resources.records.systemfields import PIDListRelation
+from invenio_records_resources.records.systemfields import (
+    ModelPIDField,
+    PIDListRelation,
+)
+from invenio_records_resources.resources.records.headers import etag_headers
+
+from invenio_vocabularies.contrib.names.permissions import NamesPermissionPolicy
 
-from ...records.pidprovider import PIDProviderFactory
-from ...records.systemfields import BaseVocabularyPIDFieldContext
-from ...services.permissions import PermissionPolicy
 from ..affiliations.api import Affiliation
 from .config import NamesSearchOptions, service_components
 from .schema import NameSchema
@@ -35,28 +39,40 @@ name_relations = RelationsField(
 record_type = RecordTypeFactory(
     "Name",
     # Data layer
+    pid_field_cls=ModelPIDField,
     pid_field_kwargs={
-        "create": False,
-        "provider": PIDProviderFactory.create(
-            pid_type="names", base_cls=RecordIdProviderV2
-        ),
-        "context_cls": BaseVocabularyPIDFieldContext,
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
     },
     schema_version="1.0.0",
     schema_path="local://names/name-v1.0.0.json",
+    index_name="names-name-v2.0.0",
     record_relations=name_relations,
     record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
         extensions=[
             RelationDumperExt("relations"),
             IndexedAtDumperExt(),
-        ]
+        ],
     ),
     # Service layer
     service_id="names",
     service_schema=NameSchema,
     search_options=NamesSearchOptions,
     service_components=service_components,
-    permission_policy_cls=PermissionPolicy,
+    permission_policy_cls=NamesPermissionPolicy,
     # Resource layer
     endpoint_route="/names",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

invenio_vocabularies/contrib/names/permissions.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary permissions."""
+
+from invenio_records_permissions.generators import AuthenticatedUser, SystemProcess
+
+from ...services.permissions import PermissionPolicy
+
+
+class NamesPermissionPolicy(PermissionPolicy):
+    """Permission policy."""
+
+    can_search = [SystemProcess(), AuthenticatedUser()]
+    can_read = [SystemProcess(), AuthenticatedUser()]

invenio_vocabularies/contrib/names/s3client.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""S3 client."""
+
+from flask import current_app
+
+try:
+    import s3fs
+except ImportError:
+    s3fs = None
+
+
+class S3Client:
+    """S3 client."""
+
+    def __init__(self, access_key, secret_key):
+        """Constructor."""
+        if s3fs is None:
+            raise Exception("s3fs is not installed.")
+
+        self.fs = s3fs.S3FileSystem(key=access_key, secret=secret_key)
+
+    def read_file(self, s3_path):
+        """Reads a file from S3."""
+        with self.fs.open(s3_path, "rb") as f:
+            return f.read()
+
+
+class S3OrcidClient(S3Client):
+    """S3 ORCiD client."""
+
+    def __init__(self):
+        """Constructor."""
+        super().__init__(
+            access_key=current_app.config["VOCABULARIES_ORCID_ACCESS_KEY"],
+            secret_key=current_app.config["VOCABULARIES_ORCID_SECRET_KEY"],
+        )

invenio_vocabularies/contrib/names/schema.py

@@ -11,16 +11,16 @@
 from functools import partial
 
 from invenio_i18n import lazy_gettext as _
-from invenio_records_resources.services.records.schema import BaseRecordSchema
-from marshmallow import ValidationError, fields, post_load, validates_schema
+from marshmallow import ValidationError, fields, post_dump, post_load, validates_schema
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
+from ...services.schema import BaseVocabularySchema, ModePIDFieldVocabularyMixin
 from ..affiliations.schema import AffiliationRelationSchema
 from .config import names_schemes
 
 
-class NameSchema(BaseRecordSchema):
+class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin):
     """Service schema for names.
 
     Note that this vocabulary is very different from the base vocabulary
@@ -56,8 +56,22 @@ class NameSchema(BaseRecordSchema):
             ]
             raise ValidationError({"family_name": messages})
 
+    @validates_schema
+    def validate_affiliatons(self, data, **kwargs):
+        """Validate names."""
+        affiliations = data.get("affiliations", [])
+        seen_names = set()
+        for affiliation in affiliations:
+            name = affiliation.get("name")
+            if not affiliation.get("id") and name:
+                if name in seen_names:
+                    messages = [_("Duplicated affiliations.")]
+                    raise ValidationError({"affiliations": messages})
+                else:
+                    seen_names.add(name)
+
     @post_load
-    def calculate_name(self, data, **kwargs):
+    def update_name(self, data, **kwargs):
         """Update names for person.
 
         Fill name from given_name and family_name if person.
@@ -68,3 +82,16 @@ class NameSchema(BaseRecordSchema):
             data["name"] = f"{family_name}, {given_name}"
 
         return data
+
+    @post_dump
+    def dump_name(self, data, **kwargs):
+        """Dumps the name if not present in the serialized data."""
+        name = data.get("name")
+        if not name:
+            family_name = data.get("family_name")
+            given_name = data.get("given_name")
+            if family_name and given_name:
+                data["name"] = f"{family_name}, {given_name}"
+            elif family_name:
+                data["name"] = family_name
+        return data

invenio_vocabularies/contrib/subjects/config.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2021 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,13 +10,19 @@
 
 """Subjects configuration."""
 
+from flask import current_app
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
+from werkzeug.local import LocalProxy
 
 from ...services.components import PIDComponent
 from ...services.querystr import FilteredSuggestQueryParser
 
+subject_schemes = LocalProxy(
+    lambda: current_app.config["VOCABULARIES_SUBJECTS_SCHEMES"]
+)
+
 
 class SubjectsSearchOptions(SearchOptions):
     """Search options."""
@@ -23,9 +30,8 @@ class SubjectsSearchOptions(SearchOptions):
     suggest_parser_cls = FilteredSuggestQueryParser.factory(
         filter_field="scheme",
         fields=[  # suggest fields
-            "subject^100",
-            "subject._2gram",
-            "subject._3gram",
+            "title.*^100",
+            "synonyms^20",
         ],
     )
 

invenio_vocabularies/contrib/subjects/datastreams.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 University of Münster.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Names datastreams, transformers, writers and readers."""
+
+from invenio_access.permissions import system_identity
+from invenio_i18n import lazy_gettext as _
+
+from ...datastreams.writers import ServiceWriter
+from .euroscivoc import datastreams as euroscivoc_datastreams
+from .mesh import datastreams as mesh_datastreams
+
+
+class SubjectsServiceWriter(ServiceWriter):
+    """Subjects Service Writer."""
+
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        service_or_name = kwargs.pop("service_or_name", "subjects")
+        super().__init__(service_or_name=service_or_name, *args, **kwargs)
+
+    def _entry_id(self, entry):
+        """Get the id from an entry."""
+        return entry["id"]
+
+
+VOCABULARIES_DATASTREAM_READERS = {
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_READERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_READERS,
+}
+"""Subjects Data Streams readers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_TRANSFORMERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_TRANSFORMERS,
+}
+"""Subjects Data Streams transformers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {
+    "subjects-service": SubjectsServiceWriter,
+    **mesh_datastreams.VOCABULARIES_DATASTREAM_WRITERS,
+    **euroscivoc_datastreams.VOCABULARIES_DATASTREAM_WRITERS,
+}
+"""Subjects Data Streams writers."""
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {"type": "yaml"},
+    ],
+    "writers": [
+        {
+            "type": "subjects-service",
+        }
+    ],
+}
+"""Data Stream configuration."""

invenio_vocabularies/contrib/subjects/euroscivoc/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""EuroSciVoc Subjects module."""

invenio_vocabularies/contrib/subjects/euroscivoc/datastreams.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""EuroSciVoc subjects datastreams, readers, transformers, and writers."""
+
+import io
+from collections import namedtuple
+
+import requests
+from rdflib import OWL, RDF, Graph, Namespace
+
+from invenio_vocabularies.config import SUBJECTS_EUROSCIVOC_FILE_URL
+from invenio_vocabularies.datastreams.readers import BaseReader
+from invenio_vocabularies.datastreams.transformers import BaseTransformer
+
+
+class EuroSciVocSubjectsHTTPReader(BaseReader):
+    """Reader class to fetch and process EuroSciVoc RDF data."""
+
+    def __init__(self, origin=None, mode="r", since=None, *args, **kwargs):
+        """Initialize the reader with the data source.
+
+        :param origin: The URL from which to fetch the RDF data.
+        :param mode: Mode of operation (default is 'r' for reading).
+        """
+        self.origin = origin or SUBJECTS_EUROSCIVOC_FILE_URL
+        super().__init__(origin=origin, mode=mode, *args, **kwargs)
+
+    def _iter(self, rdf_graph):
+        """Iterate over the RDF graph, yielding one subject at a time.
+
+        :param rdf_graph: The RDF graph to process.
+        :yield: Subject and graph to be transformed.
+        """
+        SKOS_CORE = Namespace("http://www.w3.org/2004/02/skos/core#")
+
+        for subject, _, _ in rdf_graph.triples((None, RDF.type, SKOS_CORE.Concept)):
+            yield {"subject": subject, "rdf_graph": rdf_graph}
+
+    def read(self, item=None, *args, **kwargs):
+        """Fetch and process the EuroSciVoc RDF data, yielding it one subject at a time.
+
+        :param item: The RDF data provided as bytes (optional).
+        :yield: Processed EuroSciVoc subject data.
+        """
+        if item:
+            raise NotImplementedError(
+                "EuroSciVocSubjectsHTTPReader does not support being chained after another reader"
+            )
+        # Fetch the RDF data from the specified origin URL
+        response = requests.get(self.origin)
+        response.raise_for_status()
+
+        # Treat the response content as a file-like object
+        rdf_data = io.BytesIO(response.content)
+
+        # Parse the RDF data into a graph
+        rdf_graph = Graph()
+        rdf_graph.parse(rdf_data, format="xml")
+
+        # Yield each processed subject from the RDF graph
+        yield from self._iter(rdf_graph)
+
+
+class EuroSciVocSubjectsTransformer(BaseTransformer):
+    """Transformer class to convert EuroSciVoc RDF data to a dictionary format."""
+
+    SKOS_CORE = Namespace("http://www.w3.org/2004/02/skos/core#")
+    SPLITCHAR = ","
+
+    def _get_notation(self, subject, rdf_graph):
+        """Extract the numeric notation for a subject."""
+        for _, _, notation in rdf_graph.triples(
+            (subject, self.SKOS_CORE.notation, None)
+        ):
+            if str(notation).isdigit():
+                return str(notation)
+        return None
+
+    def _get_labels(self, subject, rdf_graph):
+        """Extract prefLabel and altLabel languages for a subject."""
+        labels = {
+            label.language: label.value.capitalize()
+            for _, _, label in rdf_graph.triples(
+                (subject, self.SKOS_CORE.prefLabel, None)
+            )
+        }
+        if "en" not in labels:
+            for _, _, label in rdf_graph.triples(
+                (subject, self.SKOS_CORE.altLabel, None)
+            ):
+                labels.setdefault(label.language, label.value.capitalize())
+        return labels
+
+    def _find_parents(self, subject, rdf_graph):
+        """Find parent notations."""
+        parents = []
+
+        # Traverse the broader hierarchy
+        for broader in rdf_graph.transitive_objects(subject, self.SKOS_CORE.broader):
+            if broader != subject:  # Ensure we don't include the current subject
+                parent_notation = self._get_notation(broader, rdf_graph)
+                if parent_notation:
+                    parents.append(parent_notation)
+
+        return parents
+
+    def _transform_entry(self, subject, rdf_graph):
+        """Transform an entry to the required dictionary format."""
+        # Get subject notation with euroscivoc prefix
+        notation = self._get_notation(subject, rdf_graph)
+        id = f"euroscivoc:{notation}" if notation else None
+        # Get labels for the current subject
+        labels = self._get_labels(subject, rdf_graph)
+        # Join parent notations with SPLITCHAR separator and add euroscivoc prefix
+        parents = self.SPLITCHAR.join(
+            f"euroscivoc:{n}" for n in reversed(self._find_parents(subject, rdf_graph))
+        )
+        # Create identifiers list
+        identifiers = [{"scheme": "url", "identifier": str(subject)}]
+
+        return {
+            "id": id,
+            "scheme": "EuroSciVoc",
+            "subject": labels.get("en", "").capitalize(),
+            "title": labels,
+            "props": {"parents": parents} if parents else {},
+            "identifiers": identifiers,
+        }
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Transform a stream entry to the required dictionary format.
+
+        :param stream_entry: The entry to be transformed, which includes the subject and the RDF graph.
+        :return: The transformed stream entry.
+        """
+        # Apply transformations
+        entry_data = self._transform_entry(
+            stream_entry.entry["subject"], stream_entry.entry["rdf_graph"]
+        )
+        stream_entry.entry = entry_data
+        return stream_entry
+
+
+# Configuration for datastream readers, transformers, and writers
+VOCABULARIES_DATASTREAM_READERS = {"euroscivoc-reader": EuroSciVocSubjectsHTTPReader}
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {
+    "euroscivoc-transformer": EuroSciVocSubjectsTransformer
+}
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "euroscivoc-reader",
+        }
+    ],
+    "transformers": [{"type": "euroscivoc-transformer"}],
+    "writers": [
+        {
+            "type": "subjects-service",
+        }
+    ],
+}

invenio_vocabularies/contrib/subjects/jsonschemas/subjects/subject-v1.0.0.json

@@ -8,6 +8,9 @@
     "$schema": {
       "$ref": "local://definitions-v1.0.0.json#/$schema"
     },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
     "id": {
       "description": "URI or classification code as identifier - globally unique among all subject schemes.",
       "$ref": "local://definitions-v1.0.0.json#/identifier"
@@ -22,6 +25,34 @@
     "subject": {
       "description": "Human readable label.",
       "type": "string"
+    },
+    "title": {
+      "description": "Human readable label in different languages.",
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "props": {
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "type": "string"
+        }
+      }
+    },
+    "identifiers": {
+      "description": "Alternate identifiers for the subject.",
+      "type": "array",
+      "items": {
+        "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "synonyms": {
+      "description": "Synonyms of the subject label.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
     }
   }
 }

invenio_vocabularies/contrib/subjects/mappings/os-v1/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/os-v2/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mappings/v7/subjects/subject-v1.0.0.json

@@ -1,5 +1,16 @@
 {
   "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
     "dynamic": "strict",
     "properties": {
       "$schema": {
@@ -55,6 +66,30 @@
             "type": "keyword"
           }
         }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
       }
     }
   }

invenio_vocabularies/contrib/subjects/mesh/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH Subjects module."""