invenio-vocabularies 9.1.2__py2.py3-none-any.whl

invenio_vocabularies/factories.py

@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024-2025 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+"""Generate Vocabulary Config."""
+
+from copy import deepcopy
+
+import yaml
+from invenio_records_resources.proxies import current_service_registry
+
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG as affiliations_ds_config,
+)
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG_EDMO as affiliations_edmo_ds_config,
+)
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG_OPENAIRE as affiliations_openaire_ds_config,
+)
+from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
+from .contrib.awards.datastreams import (
+    DATASTREAM_CONFIG_CORDIS as awards_cordis_ds_config,
+)
+from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
+from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
+from .contrib.subjects.datastreams import DATASTREAM_CONFIG as subjects_ds_config
+from .contrib.subjects.euroscivoc.datastreams import (
+    DATASTREAM_CONFIG as euroscivoc_ds_config,
+)
+from .contrib.subjects.gemet.datastreams import DATASTREAM_CONFIG as gemet_ds_config
+from .contrib.subjects.nvs.datastreams import DATASTREAM_CONFIG as nvs_ds_config
+
+
+class VocabularyConfig:
+    """Vocabulary Config Factory."""
+
+    config = None
+    vocabulary_name = None
+
+    def get_config(self, filepath=None, origin=None):
+        """Get the configuration for the vocabulary."""
+        config = deepcopy(self.config)
+        if filepath:
+            with open(filepath, encoding="utf-8") as f:
+                config = yaml.safe_load(f).get(self.vocabulary_name)
+        if origin:
+            config["readers"][0].setdefault("args", {})
+            config["readers"][0]["args"]["origin"] = origin
+        return config
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        return current_service_registry.get(self.vocabulary_name)
+
+
+class NamesVocabularyConfig(VocabularyConfig):
+    """Names Vocabulary Config."""
+
+    config = names_ds_config
+    vocabulary_name = "names"
+
+
+class FundersVocabularyConfig(VocabularyConfig):
+    """Funders Vocabulary Config."""
+
+    config = funders_ds_config
+    vocabulary_name = "funders"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Funders")
+
+
+class SubjectsVocabularyConfig(VocabularyConfig):
+    """Subjects Vocabulary Config."""
+
+    config = subjects_ds_config
+    vocabulary_name = "subjects"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Subjects")
+
+
+class AwardsVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_ds_config
+    vocabulary_name = "awards"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Awards")
+
+
+class AwardsCordisVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_cordis_ds_config
+    vocabulary_name = "awards:cordis"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for CORDIS Awards")
+
+
+class AffiliationsVocabularyConfig(VocabularyConfig):
+    """Affiliations Vocabulary Config."""
+
+    config = affiliations_ds_config
+    vocabulary_name = "affiliations"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Affiliations")
+
+
+class AffiliationsOpenAIREVocabularyConfig(VocabularyConfig):
+    """OpenAIRE Affiliations Vocabulary Config."""
+
+    config = affiliations_openaire_ds_config
+    vocabulary_name = "affiliations:openaire"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for OpenAIRE Affiliations")
+
+
+class AffiliationsEDMOVocabularyConfig(VocabularyConfig):
+    """European Directory of Marine Organisations (EDMO) Affiliations Vocabulary Config."""
+
+    config = affiliations_edmo_ds_config
+    vocabulary_name = "affiliations:edmo"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for EDMO Affiliations")
+
+
+class SubjectsEuroSciVocVocabularyConfig(VocabularyConfig):
+    """EuroSciVoc Subjects Vocabulary Config."""
+
+    config = euroscivoc_ds_config
+    vocabulary_name = "subjects:euroscivoc"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for EuroSciVoc Subjects")
+
+
+class SubjectsGEMETVocabularyConfig(VocabularyConfig):
+    """GEMET Subjects Vocabulary Config."""
+
+    config = gemet_ds_config
+    vocabulary_name = "subjects:gemet"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for GEMET Subjects")
+
+
+class SubjectsNVSVocabularyConfig(VocabularyConfig):
+    """NVS Subjects Vocabulary Config."""
+
+    config = nvs_ds_config
+    vocabulary_name = "subjects:nvs"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for NVS Subjects")
+
+
+def get_vocabulary_config(vocabulary):
+    """Factory function to get the appropriate Vocabulary Config."""
+    vocab_config = {
+        "names": NamesVocabularyConfig,
+        "funders": FundersVocabularyConfig,
+        "awards": AwardsVocabularyConfig,
+        "awards:cordis": AwardsCordisVocabularyConfig,
+        "affiliations": AffiliationsVocabularyConfig,
+        "affiliations:openaire": AffiliationsOpenAIREVocabularyConfig,
+        "affiliations:edmo": AffiliationsEDMOVocabularyConfig,
+        "subjects": SubjectsVocabularyConfig,
+        "subjects:gemet": SubjectsGEMETVocabularyConfig,
+        "subjects:nvs": SubjectsNVSVocabularyConfig,
+        "subjects:euroscivoc": SubjectsEuroSciVocVocabularyConfig,
+    }
+    return vocab_config.get(vocabulary, VocabularyConfig)()

invenio_vocabularies/fixtures.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2022 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Fixtures module."""
+
+import yaml
+from invenio_access.permissions import system_identity
+
+from .datastreams.factories import DataStreamFactory
+from .proxies import current_service
+
+
+class VocabularyFixture:
+    """Vocabulary fixture."""
+
+    def __init__(self, filepath, delay=True):
+        """Constructor."""
+        self._filepath = filepath
+
+    def _load_vocabulary(self, config, delay=True, **kwargs):
+        """Given an entry from the vocabularies.yaml file, load its content."""
+        datastream = DataStreamFactory.create(
+            readers_config=config["readers"],
+            transformers_config=config.get("transformers"),
+            writers_config=config["writers"],
+            batch_size=config.get("batch_size", 1000),
+            write_many=config.get("write_many", False),
+        )
+
+        errors = []
+        for result in datastream.process():
+            if result.errors:
+                errors.append(result)
+
+        return errors
+
+    def _create_vocabulary(self, id_, pid_type, *args, **kwargs):
+        """Creates a vocabulary."""
+        return current_service.create_type(system_identity, id_, pid_type)
+
+    def load(self, *args, **kwargs):
+        """Return content of vocabularies file."""
+        with open(self._filepath) as f:
+            data = yaml.safe_load(f) or {}
+            for id_, config in data.items():
+                self._create_vocabulary(id_, config["pid-type"])
+                yield self._load_vocabulary(config)

invenio_vocabularies/jobs.py

@@ -0,0 +1,207 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Jobs module."""
+
+from datetime import datetime, timedelta
+
+from flask import current_app
+from invenio_i18n import lazy_gettext as _
+from invenio_jobs.jobs import JobType
+
+from invenio_vocabularies.services.tasks import process_datastream
+
+from .contrib.names.datastreams import ORCID_PRESET_DATASTREAM_CONFIG
+
+
+class ProcessDataStreamJob(JobType):
+    """Generic process data stream job type."""
+
+    task = process_datastream
+
+
+class ProcessRORAffiliationsJob(ProcessDataStreamJob):
+    """Process ROR affiliations datastream registered task."""
+
+    description = _("Process ROR affiliations")
+    title = _("Load ROR affiliations")
+    id = "process_ror_affiliations"
+
+    @classmethod
+    def build_task_arguments(cls, job_obj, since=None, **kwargs):
+        """Process ROR affiliations."""
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": since},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "-ror-data\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "affiliations-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-affiliations"}],
+            }
+        }
+
+
+class ProcessRORFundersJob(ProcessDataStreamJob):
+    """Process ROR funders datastream registered task."""
+
+    description = _("Process ROR funders")
+    title = _("Load ROR funders")
+    id = "process_ror_funders"
+
+    @classmethod
+    def build_task_arguments(cls, job_obj, since=None, **kwargs):
+        """Process ROR funders."""
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": since},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "-ror-data\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "funders-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-funders"}],
+            }
+        }
+
+
+class ImportAwardsOpenAIREJob(ProcessDataStreamJob):
+    """Import awards from OpenAIRE registered task."""
+
+    description = _("Import awards from OpenAIRE")
+    title = _("Import Awards OpenAIRE")
+    id = "import_awards_openaire"
+
+    @classmethod
+    def build_task_arguments(cls, job_obj, since=None, **kwargs):
+        """Process awards OpenAIRE."""
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "type": "openaire-http",
+                        "args": {"origin": "diff", "tar_href": "/project.tar"},
+                    },
+                    {
+                        "type": "tar",
+                        "args": {
+                            "mode": "r",
+                            "regex": "\\.json.gz$",
+                        },
+                    },
+                    {"type": "gzip"},
+                    {"type": "jsonl"},
+                ],
+                "transformers": [{"type": "openaire-award"}],
+                "writers": [
+                    {"args": {"writer": {"type": "awards-service"}}, "type": "async"}
+                ],
+            }
+        }
+
+
+class UpdateAwardsCordisJob(ProcessDataStreamJob):
+    """Update awards from CORDIS registered task."""
+
+    description = _("Update awards from CORDIS")
+    title = _("Update Awards CORDIS")
+    id = "update_awards_cordis"
+
+    @classmethod
+    def build_task_arguments(cls, job_obj, since=None, **kwargs):
+        """Process awards Cordis."""
+        return {
+            "config": {
+                "readers": [
+                    {"args": {"origin": "HE"}, "type": "cordis-project-http"},
+                    {"args": {"mode": "r", "regex": "\\.xml$"}, "type": "zip"},
+                    {"args": {"root_element": "project"}, "type": "xml"},
+                ],
+                "transformers": [{"type": "cordis-award"}],
+                "writers": [
+                    {
+                        "args": {"writer": {"type": "cordis-awards-service"}},
+                        "type": "async",
+                    }
+                ],
+            }
+        }
+
+
+class ImportORCIDJob(ProcessDataStreamJob):
+    """Import ORCID data registered task."""
+
+    description = _("Import ORCID data")
+    title = _("Import ORCID data")
+    id = "import_orcid"
+
+    @classmethod
+    def build_task_arguments(cls, job_obj, since=None, **kwargs):
+        """Process ORCID data."""
+        task_args = {"config": {**ORCID_PRESET_DATASTREAM_CONFIG}}
+        for reader in task_args["config"]["readers"]:
+            # Assign since to all readers of the ORCID job
+            # It is the responsibility of the reader to handle it or ignore it
+            reader["args"] = {**reader.get("args", {}), "since": str(since)}
+        return task_args
+
+    @classmethod
+    def _build_task_arguments(cls, job_obj, since=None, custom_args=None, **kwargs):
+        """Build dict of arguments injected on task execution.
+
+        :param job_obj (Job): the Job object.
+        :param since (datetime): last time the job was executed.
+        :param custom_args (dict): when provided, takes precedence over any other
+            provided argument.
+        :return: a dict of arguments to be injected on task execution.
+        """
+        if custom_args:
+            return custom_args
+
+        if since is None:
+            """We set since to a time in the past defined by the VOCABULARIES_ORCID_SYNC_SINCE."""
+
+            since = datetime.now() - timedelta(
+                **current_app.config["VOCABULARIES_ORCID_SYNC_SINCE"]
+            )
+        """
+        Otherwise, since is already specified as a datetime with a timezone (see PredefinedArgsSchema) or we have never
+        run the job before so there is no logical value.
+        """
+        return {**cls.build_task_arguments(job_obj, since=since, **kwargs)}

invenio_vocabularies/proxies.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+# Copyright (C) 2021 Northwestern University.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Proxies to the service and resource."""
+
+from flask import current_app
+from werkzeug.local import LocalProxy
+
+
+def _ext_proxy(attr):
+    return LocalProxy(
+        lambda: getattr(current_app.extensions["invenio-vocabularies"], attr)
+    )
+
+
+current_service = _ext_proxy("vocabularies_service")
+"""Proxy to the instantiated vocabulary service."""
+
+
+current_resource = _ext_proxy("resource")
+"""Proxy to the instantiated vocabulary resource."""

invenio_vocabularies/records/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Records directory."""

invenio_vocabularies/records/api.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Vocabulary data API."""
+
+from invenio_records.dumpers import SearchDumper
+from invenio_records.dumpers.indexedat import IndexedAtDumperExt
+from invenio_records.systemfields import ConstantField, RelatedModelField
+from invenio_records_resources.records.api import Record
+from invenio_records_resources.records.systemfields import IndexField, PIDField
+
+from .models import VocabularyMetadata, VocabularyType
+from .pidprovider import VocabularyIdProvider
+from .systemfields import VocabularyPIDFieldContext
+
+
+class Vocabulary(Record):
+    """A generic vocabulary record."""
+
+    # Configuration
+    model_cls = VocabularyMetadata
+
+    # System fields
+    # TODO: Can schema name be changed (remove localhost)
+    schema = ConstantField(
+        "$schema",
+        "local://vocabularies/vocabulary-v1.0.0.json",
+    )
+
+    index = IndexField("vocabularies-vocabulary-v1.0.0", search_alias="vocabularies")
+
+    #: Disable the metadata system field.
+    metadata = None
+
+    type = RelatedModelField(VocabularyType, required=True)
+
+    pid = PIDField(
+        "id",
+        provider=VocabularyIdProvider,
+        context_cls=VocabularyPIDFieldContext,
+        create=False,
+    )
+
+    dumper = SearchDumper(
+        extensions=[
+            IndexedAtDumperExt(),
+        ]
+    )

invenio_vocabularies/records/jsonschemas/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""JSONSchema directory."""

invenio_vocabularies/records/jsonschemas/vocabularies/definitions-v1.0.0.json

@@ -0,0 +1,30 @@
+{
+  "title": {
+    "type": "object",
+    "description": "Title of vocabulary item. Keys are locale codes.",
+    "patternProperties": {
+      "^[a-z]{2}$": {
+        "type": "string"
+      }
+    }
+  },
+  "description": {
+    "type": "object",
+    "description": "Description of vocabulary item. Keys are locale codes.",
+    "patternProperties": {
+      "^[a-z]{2}$": {
+        "type": "string"
+      }
+    }
+  },
+  "icon": {
+    "type": "string"
+  },
+  "tags": {
+    "type": "array",
+    "description": "Tags for a vocabulary item.",
+    "items": {
+      "type": "string"
+    }
+  }
+}

invenio_vocabularies/records/jsonschemas/vocabularies/vocabulary-v1.0.0.json

@@ -0,0 +1,55 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "local://vocabularies/vocabulary-v1.0.0.json",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "$schema": {
+      "$ref": "local://definitions-v1.0.0.json#/$schema"
+    },
+    "description": {
+      "type": "object",
+      "description": "Description of vocabulary item. Key are locale codes.",
+      "patternProperties": {
+        "^[a-z]{2}$": {
+          "type": "string"
+        }
+      }
+    },
+    "icon": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/icon"
+    },
+    "id": {
+      "$ref": "local://definitions-v1.0.0.json#/identifier"
+    },
+    "pid": {
+      "$ref": "local://definitions-v1.0.0.json#/internal-pid"
+    },
+    "props": {
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "type": "string"
+        }
+      }
+    },
+    "tags": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
+    },
+    "title": {
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"
+    },
+    "type": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "pid_type": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

invenio_vocabularies/records/mappings/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Mappings directory."""

invenio_vocabularies/records/mappings/os-v1/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2021 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""OpenSearch v1 mappings."""