invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/datastreams/writers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -13,18 +13,25 @@ from pathlib import Path
 
 import yaml
 from invenio_access.permissions import system_identity
-from invenio_pidstore.errors import PIDAlreadyExists
+from invenio_pidstore.errors import PIDAlreadyExists, PIDDoesNotExistError
 from invenio_records.systemfields.relations.errors import InvalidRelationValue
 from invenio_records_resources.proxies import current_service_registry
 from marshmallow import ValidationError
+from sqlalchemy.exc import NoResultFound
 
 from .datastreams import StreamEntry
 from .errors import WriterError
+from .tasks import write_entry, write_many_entry
 
 
 class BaseWriter(ABC):
     """Base writer."""
 
+    def __init__(self, *args, **kwargs):
+        """Base initialization logic."""
+        # Add any base initialization here if needed
+        pass
+
     @abstractmethod
     def write(self, stream_entry, *args, **kwargs):
         """Writes the input stream entry to the target output.
@@ -35,16 +42,29 @@ class BaseWriter(ABC):
         """
         pass
 
+    @abstractmethod
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input streams entry to the target output.
+
+        :returns: A List of StreamEntry. The result of writing the entry.
+                  Raises WriterException in case of errors.
+
+        """
+        pass
+
 
 class ServiceWriter(BaseWriter):
     """Writes the entries to an RDM instance using a Service object."""
 
-    def __init__(self, service_or_name, *args, identity=None, update=False, **kwargs):
+    def __init__(
+        self, service_or_name, *args, identity=None, insert=True, update=False, **kwargs
+    ):
         """Constructor.
 
         :param service_or_name: a service instance or a key of the
                                 service registry.
         :param identity: access identity.
+        :param insert: if True it will insert records which do not exist.
         :param update: if True it will update records if they exist.
         """
         if isinstance(service_or_name, str):
@@ -52,6 +72,7 @@ class ServiceWriter(BaseWriter):
 
         self._service = service_or_name
         self._identity = identity or system_identity
+        self._insert = insert
         self._update = update
 
         super().__init__(*args, **kwargs)
@@ -63,20 +84,47 @@ class ServiceWriter(BaseWriter):
     def _resolve(self, id_):
         return self._service.read(self._identity, id_)
 
+    def _do_update(self, entry):
+        vocab_id = self._entry_id(entry)
+        current = self._resolve(vocab_id)
+        combined_dict = current.to_dict()
+
+        # Update fields from entry
+        for key, value in entry.items():
+            if key in combined_dict:
+                if isinstance(combined_dict[key], list) and isinstance(value, list):
+                    combined_dict[key].extend(
+                        item for item in value if item not in combined_dict[key]
+                    )
+                else:
+                    combined_dict[key] = value
+            else:
+                combined_dict[key] = value
+
+        return StreamEntry(
+            self._service.update(self._identity, vocab_id, combined_dict)
+        )
+
     def write(self, stream_entry, *args, **kwargs):
         """Writes the input entry using a given service."""
         entry = stream_entry.entry
+
         try:
-            try:
-                return StreamEntry(self._service.create(self._identity, entry))
-            except PIDAlreadyExists:
-                if not self._update:
-                    raise WriterError([f"Vocabulary entry already exists: {entry}"])
-                vocab_id = self._entry_id(entry)
-                current = self._resolve(vocab_id)
-                updated = dict(current.to_dict(), **entry)
-                return StreamEntry(
-                    self._service.update(self._identity, vocab_id, updated)
+            if self._insert:
+                try:
+                    return StreamEntry(self._service.create(self._identity, entry))
+                except PIDAlreadyExists:
+                    if not self._update:
+                        raise WriterError([f"Vocabulary entry already exists: {entry}"])
+                    return self._do_update(entry)
+            elif self._update:
+                try:
+                    return self._do_update(entry)
+                except (NoResultFound, PIDDoesNotExistError):
+                    raise WriterError([f"Vocabulary entry does not exist: {entry}"])
+            else:
+                raise WriterError(
+                    ["Writer wrongly configured to not insert and to not update"]
                 )
 
         except ValidationError as err:
@@ -85,6 +133,25 @@ class ServiceWriter(BaseWriter):
             # TODO: Check if we can get the error message easier
             raise WriterError([{"InvalidRelationValue": err.args[0]}])
 
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the input entries using a given service."""
+        entries = [entry.entry for entry in stream_entries]
+        entries_with_id = [(self._entry_id(entry), entry) for entry in entries]
+        results = self._service.create_or_update_many(self._identity, entries_with_id)
+        stream_entries_processed = []
+        for entry, result in zip(entries, results):
+            processed_stream_entry = StreamEntry(
+                entry=entry,
+                record=result.record,
+                errors=result.errors,
+                op_type=result.op_type,
+                exc=result.exc,
+            )
+            processed_stream_entry.log_errors()
+            stream_entries_processed.append(processed_stream_entry)
+
+        return stream_entries_processed
+
 
 class YamlWriter(BaseWriter):
     """Writes the entries to a YAML file."""
@@ -103,6 +170,41 @@ class YamlWriter(BaseWriter):
         with open(self._filepath, "a") as file:
             # made into array for safer append
             # will always read array (good for reader)
-            yaml.safe_dump([stream_entry.entry], file)
+            yaml.safe_dump([stream_entry.entry], file, allow_unicode=True)
 
         return stream_entry
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Writes the yaml input entries."""
+        with open(self._filepath, "a") as file:
+            yaml.safe_dump(
+                [stream_entry.entry for stream_entry in stream_entries],
+                file,
+                allow_unicode=True,
+            )
+
+
+class AsyncWriter(BaseWriter):
+    """Writes the entries asynchronously (celery task)."""
+
+    def __init__(self, writer, *args, **kwargs):
+        """Constructor.
+
+        :param writer: writer to use.
+        """
+        super().__init__(*args, **kwargs)
+        self._writer = writer
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_entry.delay(self._writer, stream_entry.entry)
+
+        return stream_entry
+
+    def write_many(self, stream_entries, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_many_entry.delay(
+            self._writer, [stream_entry.entry for stream_entry in stream_entries]
+        )
+
+        return stream_entries

invenio_vocabularies/datastreams/xml.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""XML utils."""
+
+from collections import defaultdict
+
+
+def etree_to_dict(tree):
+    """Convert an ElementTree to a dictionary."""
+    tag = tree.tag.split(":")[-1]  # strip namespace
+    d = {tag: {} if tree.attrib else None}
+    children = list(tree)
+    if children:
+        dd = defaultdict(list)
+        for dc in map(etree_to_dict, children):
+            for k, v in dc.items():
+                dd[k].append(v)
+        d = {tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
+    if tree.attrib:
+        d[tag].update(("@" + k, v) for k, v in tree.attrib.items())
+    if tree.text:
+        text = tree.text.strip()
+        if children or tree.attrib:
+            if text:
+                d[tag]["#text"] = text
+        else:
+            d[tag] = text
+    return d

invenio_vocabularies/ext.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2023 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -39,8 +40,14 @@ from .contrib.subjects import (
     SubjectsService,
     SubjectsServiceConfig,
 )
-from .resources.resource import VocabulariesResource
-from .services.service import VocabulariesService
+from .resources import (
+    VocabulariesAdminResource,
+    VocabulariesResource,
+    VocabulariesResourceConfig,
+    VocabularyTypeResourceConfig,
+)
+from .services.config import VocabularyTypesServiceConfig
+from .services.service import VocabulariesService, VocabularyTypeService
 
 
 class InvenioVocabularies(object):
@@ -75,6 +82,7 @@ class InvenioVocabularies(object):
             funders = FundersServiceConfig
             names = NamesServiceConfig
             subjects = SubjectsServiceConfig
+            vocabulary_types = VocabularyTypesServiceConfig
 
         return ServiceConfigs
 
@@ -92,9 +100,12 @@ class InvenioVocabularies(object):
         self.funders_service = FundersService(config=service_configs.funders)
         self.names_service = NamesService(config=service_configs.names)
         self.subjects_service = SubjectsService(config=service_configs.subjects)
-        self.service = VocabulariesService(
+        self.vocabularies_service = VocabulariesService(
             config=app.config["VOCABULARIES_SERVICE_CONFIG"],
         )
+        self.vocabulary_types_service = VocabularyTypeService(
+            config=service_configs.vocabulary_types
+        )
 
     def init_resource(self, app):
         """Initialize vocabulary resources."""
@@ -120,6 +131,49 @@ class InvenioVocabularies(object):
             config=SubjectsResourceConfig,
         )
         self.resource = VocabulariesResource(
-            service=self.service,
+            service=self.vocabularies_service,
             config=app.config["VOCABULARIES_RESOURCE_CONFIG"],
         )
+        self.vocabulary_admin_resource = VocabulariesAdminResource(
+            service=self.vocabulary_types_service,
+            config=VocabularyTypeResourceConfig,
+        )
+
+
+def finalize_app(app):
+    """Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def api_finalize_app(app):
+    """Api Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def init(app):
+    """Init app."""
+    # Register services - cannot be done in extension because
+    # Invenio-Records-Resources might not have been initialized.
+    sregistry = app.extensions["invenio-records-resources"].registry
+    ext = app.extensions["invenio-vocabularies"]
+    sregistry.register(ext.affiliations_service, service_id="affiliations")
+    sregistry.register(ext.awards_service, service_id="awards")
+    sregistry.register(ext.funders_service, service_id="funders")
+    sregistry.register(ext.names_service, service_id="names")
+    sregistry.register(ext.subjects_service, service_id="subjects")
+    sregistry.register(ext.vocabularies_service, service_id="vocabularies")
+    sregistry.register(ext.vocabulary_types_service, service_id="vocabulary-types")
+    # Register indexers
+    iregistry = app.extensions["invenio-indexer"].registry
+    iregistry.register(ext.affiliations_service.indexer, indexer_id="affiliations")
+    iregistry.register(ext.awards_service.indexer, indexer_id="awards")
+    iregistry.register(ext.funders_service.indexer, indexer_id="funders")
+    iregistry.register(ext.names_service.indexer, indexer_id="names")
+    iregistry.register(ext.subjects_service.indexer, indexer_id="subjects")
+    iregistry.register(ext.vocabularies_service.indexer, indexer_id="vocabularies")

invenio_vocabularies/factories.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+"""Generate Vocabulary Config."""
+
+from copy import deepcopy
+
+import yaml
+from invenio_records_resources.proxies import current_service_registry
+
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG as affiliations_ds_config,
+)
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG_OPENAIRE as affiliations_openaire_ds_config,
+)
+from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
+from .contrib.awards.datastreams import (
+    DATASTREAM_CONFIG_CORDIS as awards_cordis_ds_config,
+)
+from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
+from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
+from .contrib.subjects.datastreams import DATASTREAM_CONFIG as subjects_ds_config
+
+
+class VocabularyConfig:
+    """Vocabulary Config Factory."""
+
+    config = None
+    vocabulary_name = None
+
+    def get_config(self, filepath=None, origin=None):
+        """Get the configuration for the vocabulary."""
+        config = deepcopy(self.config)
+        if filepath:
+            with open(filepath, encoding="utf-8") as f:
+                config = yaml.safe_load(f).get(self.vocabulary_name)
+        if origin:
+            config["readers"][0].setdefault("args", {})
+            config["readers"][0]["args"]["origin"] = origin
+        return config
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        return current_service_registry.get(self.vocabulary_name)
+
+
+class NamesVocabularyConfig(VocabularyConfig):
+    """Names Vocabulary Config."""
+
+    config = names_ds_config
+    vocabulary_name = "names"
+
+
+class FundersVocabularyConfig(VocabularyConfig):
+    """Funders Vocabulary Config."""
+
+    config = funders_ds_config
+    vocabulary_name = "funders"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Funders")
+
+
+class SubjectsVocabularyConfig(VocabularyConfig):
+    """Subjects Vocabulary Config."""
+
+    config = subjects_ds_config
+    vocabulary_name = "subjects"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Subjects")
+
+
+class AwardsVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_ds_config
+    vocabulary_name = "awards"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Awards")
+
+
+class AwardsCordisVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_cordis_ds_config
+    vocabulary_name = "awards:cordis"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for CORDIS Awards")
+
+
+class AffiliationsVocabularyConfig(VocabularyConfig):
+    """Affiliations Vocabulary Config."""
+
+    config = affiliations_ds_config
+    vocabulary_name = "affiliations"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Affiliations")
+
+
+class AffiliationsOpenAIREVocabularyConfig(VocabularyConfig):
+    """OpenAIRE Affiliations Vocabulary Config."""
+
+    config = affiliations_openaire_ds_config
+    vocabulary_name = "affiliations:openaire"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for OpenAIRE Affiliations")
+
+
+def get_vocabulary_config(vocabulary):
+    """Factory function to get the appropriate Vocabulary Config."""
+    vocab_config = {
+        "names": NamesVocabularyConfig,
+        "funders": FundersVocabularyConfig,
+        "awards": AwardsVocabularyConfig,
+        "awards:cordis": AwardsCordisVocabularyConfig,
+        "affiliations": AffiliationsVocabularyConfig,
+        "affiliations:openaire": AffiliationsOpenAIREVocabularyConfig,
+        "subjects": SubjectsVocabularyConfig,
+    }
+    return vocab_config.get(vocabulary, VocabularyConfig)()

invenio_vocabularies/jobs.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Jobs module."""
+
+import datetime
+from datetime import timezone
+
+from invenio_i18n import gettext as _
+from invenio_jobs.jobs import JobType
+from marshmallow import Schema, fields
+from marshmallow_utils.fields import TZDateTime
+
+from invenio_vocabularies.services.tasks import process_datastream
+
+
+class ArgsSchema(Schema):
+    """Schema of task input arguments."""
+
+    since = TZDateTime(
+        timezone=timezone.utc,
+        format="iso",
+        metadata={
+            "description": _(
+                "YYYY-MM-DD HH:mm format. "
+                "Leave field empty if it should continue since last successful run."
+            )
+        },
+    )
+    job_arg_schema = fields.String(
+        metadata={"type": "hidden"},
+        dump_default="ArgsSchema",
+        load_default="ArgsSchema",
+    )
+
+
+class ProcessDataStreamJob(JobType):
+    """Generic process data stream job type."""
+
+    arguments_schema = ArgsSchema
+    task = process_datastream
+    id = None
+
+
+class ProcessRORAffiliationsJob(ProcessDataStreamJob):
+    """Process ROR affiliations datastream registered task."""
+
+    description = "Process ROR affiliations"
+    title = "Load ROR affiliations"
+    id = "process_ror_affiliations"
+
+    @classmethod
+    def default_args(cls, job_obj, since=None, **kwargs):
+        """Generate default job arguments here."""
+        if since is None and job_obj.last_runs["success"]:
+            since = job_obj.last_runs["success"].started_at
+        else:
+            since = since or datetime.datetime.now()
+
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": str(since)},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "_schema_v2\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "affiliations-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-affiliations"}],
+            }
+        }
+
+
+class ProcessRORFundersJob(ProcessDataStreamJob):
+    """Process ROR funders datastream registered task."""
+
+    description = "Process ROR funders"
+    title = "Load ROR funders"
+    id = "process_ror_funders"
+
+    @classmethod
+    def default_args(cls, job_obj, since=None, **kwargs):
+        """Generate default job arguments here."""
+        if since is None and job_obj.last_runs["success"]:
+            since = job_obj.last_runs["success"].started_at
+        else:
+            since = since or datetime.datetime.now()
+
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": str(since)},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "_schema_v2\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "funders-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-funders"}],
+            }
+        }

invenio_vocabularies/proxies.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -19,7 +19,7 @@ def _ext_proxy(attr):
     )
 
 
-current_service = _ext_proxy("service")
+current_service = _ext_proxy("vocabularies_service")
 """Proxy to the instantiated vocabulary service."""
 
 

invenio_vocabularies/records/jsonschemas/vocabularies/definitions-v1.0.0.json

@@ -10,5 +10,12 @@
   },
   "icon": {
     "type": "string"
+  },
+  "tags": {
+    "type": "array",
+    "description": "Tags for a vocabulary item.",
+    "items": {
+      "type": "string"
+    }
   }
 }

invenio_vocabularies/records/jsonschemas/vocabularies/vocabulary-v1.0.0.json

@@ -34,10 +34,7 @@
       }
     },
     "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
     },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"

invenio_vocabularies/records/mappings/os-v1/vocabularies/vocabulary-v1.0.0.json

@@ -81,7 +81,7 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true,
+        "dynamic": "true",
         "properties": {
           "en": {
             "type": "search_as_you_type",
@@ -91,7 +91,7 @@
       },
       "description": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       },
       "icon": {
         "type": "keyword",
@@ -102,7 +102,7 @@
       },
       "props": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       }
     }
   }