invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/contrib/subjects/gemet/datastreams.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""GEMET subjects datastreams, readers, transformers, and writers."""
+
+from invenio_vocabularies.datastreams.transformers import RDFTransformer
+
+from ..config import gemet_file_url
+
+# Available with the "rdf" extra
+try:
+    import rdflib
+except ImportError:
+    rdflib = None
+
+
+class GEMETSubjectsTransformer(RDFTransformer):
+    """
+    Transformer class to convert GEMET RDF data to a dictionary format.
+
+    Input:
+        - Relevant fields:
+            - `skos:prefLabel`: Preferred labels with language codes.
+            - `skos:broader`: References to broader concepts (parent concepts).
+            - `skos:memberOf`: References to groups or themes the concept belongs to.
+
+    Output:
+        - A dictionary with the following structure:
+            {
+                "id": "gemet:concept/10008",  # GEMET-specific concept ID (skos:Concept).
+                "scheme": "GEMET",  # The scheme name indicating this is a GEMET concept.
+                "subject": "Consumer product",  # The subject label (first preferred label in English, skos:prefLabel).
+                "title": {
+                    "en": "Consumer product",  # English label for the concept (skos:prefLabel).
+                    "ar": "منتج استهلاكي"  # Arabic label for the concept (skos:prefLabel).
+                },
+                "props": {
+                    "parents": "gemet:concept/6660",  # The parent concept (skos:broader), identified by its GEMET Concept ID.
+                    "groups": ["http://www.eionet.europa.eu/gemet/group/10112"],  # Group the concept belongs to (skos:memberOf)(skos:prefLabel).
+                    "themes": [
+                        "http://www.eionet.europa.eu/gemet/theme/27",  # Theme the concept belongs to (skos:memberOf)(rdfs:label).
+                    ]
+                },
+                "identifiers": [
+                    {
+                        "scheme": "url",  # Type of identifier (URL).
+                        "identifier": "http://www.eionet.europa.eu/gemet/concept/10008"  # URI of the concept (rdf:about).
+                    }
+                ]
+            }
+    """
+
+    def _get_parent_notation(self, broader, rdf_graph):
+        """Extract parent notation from GEMET URI."""
+        return "/".join(broader.split("/")[-2:])
+
+    def _get_groups_and_themes(self, subject, rdf_graph):
+        """Extract groups and themes for a subject."""
+        groups = []
+        themes = []
+
+        for relation in rdf_graph.subjects(
+            predicate=self.skos_core.member, object=subject
+        ):
+            relation_uri = str(relation)
+            relation_label = None
+
+            # If the relation is a group, check for skos:prefLabel
+            if "group" in relation_uri:
+                labels = rdf_graph.objects(
+                    subject=relation, predicate=self.skos_core.prefLabel
+                )
+                relation_label = next(
+                    (str(label) for label in labels if label.language == "en"), None
+                )
+                groups.append(relation_uri)
+
+            # If the relation is a theme, check for rdfs:label
+            elif "theme" in relation_uri:
+                labels = rdf_graph.objects(
+                    subject=relation, predicate=rdflib.RDFS.label
+                )
+                relation_label = next(
+                    (str(label) for label in labels if label.language == "en"), None
+                )
+                themes.append(relation_uri)
+
+        return groups, themes
+
+    def _transform_entry(self, subject, rdf_graph):
+        """Transform an entry to the required dictionary format."""
+        concept_number = "/".join(subject.split("/")[-2:])
+        id = f"gemet:{concept_number}" if concept_number else None
+        labels = self._get_labels(subject, rdf_graph)
+        parents = ",".join(
+            f"gemet:{n}" for n in reversed(self._find_parents(subject, rdf_graph)) if n
+        )
+        identifiers = [{"scheme": "url", "identifier": str(subject)}]
+        groups, themes = self._get_groups_and_themes(subject, rdf_graph)
+
+        props = {"parents": parents} if parents else {}
+
+        if groups:
+            props["groups"] = groups
+        if themes:
+            props["themes"] = themes
+
+        return {
+            "id": id,
+            "scheme": "GEMET",
+            "subject": labels.get("en", "").capitalize(),
+            "title": labels,
+            "props": props,
+            "identifiers": self._get_identifiers(subject),
+        }
+
+
+# Configuration for datastream
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"gemet-transformer": GEMETSubjectsTransformer}
+
+DATASTREAM_CONFIG = {
+    "readers": [
+        {
+            "type": "http",
+            "args": {
+                "origin": gemet_file_url,
+            },
+        },
+        {"type": "gzip"},
+        {"type": "rdf"},
+    ],
+    "transformers": [{"type": "gemet-transformer"}],
+    "writers": [{"args": {"writer": {"type": "subjects-service"}}, "type": "async"}],
+}

invenio_vocabularies/contrib/subjects/jsonschemas/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""JSONSchema directory."""

invenio_vocabularies/contrib/subjects/jsonschemas/subjects/subject-v1.0.0.json

@@ -0,0 +1,69 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "local://subjects/subject-v1.0.0.json",
+  "description": "Subjects from specific vocabularies.",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "$schema": {
+      "$ref": "local://definitions-v1.0.0.json#/$schema"
+    },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
+    "id": {
+      "description": "URI or classification code as identifier - globally unique among all subject schemes.",
+      "$ref": "local://definitions-v1.0.0.json#/identifier"
+    },
+    "pid": {
+      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
+    },
+    "scheme": {
+      "description": "Identifier of the subject scheme.",
+      "$ref": "local://definitions-v1.0.0.json#/identifier"
+    },
+    "subject": {
+      "description": "Human readable label.",
+      "type": "string"
+    },
+    "title": {
+      "description": "Human readable label in different languages.",
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "props": {
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ]
+        }
+      },
+      "additionalProperties": false
+    },
+    "identifiers": {
+      "description": "Alternate identifiers for the subject.",
+      "type": "array",
+      "items": {
+        "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
+      },
+      "uniqueItems": true
+    },
+    "synonyms": {
+      "description": "Synonyms of the subject label.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    }
+  }
+}

invenio_vocabularies/contrib/subjects/mappings/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Mapping directory."""

invenio_vocabularies/contrib/subjects/mappings/os-v1/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""OpenSearch version 1 mappings."""

invenio_vocabularies/contrib/subjects/mappings/os-v1/subjects/subject-v1.0.0.json

@@ -0,0 +1,96 @@
+{
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "scheme": {
+        "type": "keyword"
+      },
+      "subject": {
+        "type": "text",
+        "copy_to": "subject_sort",
+        "fields": {
+          "suggest": {
+            "type": "search_as_you_type"
+          }
+        }
+      },
+      "subject_sort": {
+        "type": "keyword"
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/subjects/mappings/os-v2/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""OpenSearch version 2 mappings."""

invenio_vocabularies/contrib/subjects/mappings/os-v2/subjects/subject-v1.0.0.json

@@ -0,0 +1,96 @@
+{
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "scheme": {
+        "type": "keyword"
+      },
+      "subject": {
+        "type": "text",
+        "copy_to": "subject_sort",
+        "fields": {
+          "suggest": {
+            "type": "search_as_you_type"
+          }
+        }
+      },
+      "subject_sort": {
+        "type": "keyword"
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/subjects/mappings/v7/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Elasticsearch version 7 mappings."""

invenio_vocabularies/contrib/subjects/mappings/v7/subjects/subject-v1.0.0.json

@@ -0,0 +1,96 @@
+{
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "i18n_title": {
+          "path_match": "title.*",
+          "match_mapping_type": "string",
+          "mapping": {
+            "type": "search_as_you_type"
+          }
+        }
+      }
+    ],
+    "dynamic": "strict",
+    "properties": {
+      "$schema": {
+        "type": "keyword",
+        "index": "false"
+      },
+      "created": {
+        "type": "date"
+      },
+      "updated": {
+        "type": "date"
+      },
+      "indexed_at": {
+        "type": "date"
+      },
+      "uuid": {
+        "type": "keyword"
+      },
+      "version_id": {
+        "type": "integer"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "scheme": {
+        "type": "keyword"
+      },
+      "subject": {
+        "type": "text",
+        "copy_to": "subject_sort",
+        "fields": {
+          "suggest": {
+            "type": "search_as_you_type"
+          }
+        }
+      },
+      "subject_sort": {
+        "type": "keyword"
+      },
+      "pid": {
+        "type": "object",
+        "properties": {
+          "pk": {
+            "type": "integer"
+          },
+          "pid_type": {
+            "type": "keyword"
+          },
+          "obj_type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "props": {
+        "type": "object",
+        "dynamic": "true"
+      },
+      "identifiers": {
+        "properties": {
+          "identifier": {
+            "type": "keyword"
+          },
+          "scheme": {
+            "type": "keyword"
+          }
+        }
+      },
+      "synonyms": {
+        "type": "text"
+      },
+      "tags": {
+        "type": "keyword"
+      }
+    }
+  }
+}

invenio_vocabularies/contrib/subjects/mesh/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH Subjects module."""

invenio_vocabularies/contrib/subjects/mesh/datastreams.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+# Copyright (C) 2024 California Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""MeSH subjects datastreams, transformers, writers and readers."""
+
+from invenio_vocabularies.datastreams.transformers import (
+    BaseTransformer,
+    TransformerError,
+)
+
+
+class MeshSubjectsTransformer(BaseTransformer):
+    """MeSH subjects Transformer."""
+
+    def apply(self, stream_entry, *args, **kwargs):
+        """Apply transformation on steam entry."""
+        entry_data = stream_entry.entry
+
+        # ID in MeSH data is in the URL, ex. https://id.nlm.nih.gov/mesh/D000001
+        # We just want to use the ID prefixed by "mesh:""
+        try:
+            mesh_id = entry_data["id"].split("/")[-1]
+            entry_data["id"] = "mesh:" + mesh_id
+        except Exception:
+            raise TransformerError("Not a valid MeSH ID.")
+
+        entry_data["title"] = title = entry_data.get("title", {})
+        # NOTE: MeSH import file comes with an English subject by default
+        if "en" not in title:
+            title["en"] = entry_data["subject"]
+
+        return stream_entry
+
+
+VOCABULARIES_DATASTREAM_READERS = {}
+"""MeSH datastream readers."""
+
+VOCABULARIES_DATASTREAM_WRITERS = {}
+"""MeSH subject datastream writers."""
+
+VOCABULARIES_DATASTREAM_TRANSFORMERS = {"mesh-subjects": MeshSubjectsTransformer}
+"""MeSH subjects datastream transformers."""

invenio_vocabularies/contrib/subjects/models.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary subjects."""
+
+from .subjects import record_type
+
+SubjectsMetadata = record_type.model_cls

invenio_vocabularies/contrib/subjects/nvs/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024-2025 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""NVS Subjects module."""