invenio-vocabularies 2.3.1__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/datastreams/xml.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""XML utils."""
+
+from collections import defaultdict
+
+
+def etree_to_dict(tree):
+    """Convert an ElementTree to a dictionary."""
+    tag = tree.tag.split(":")[-1]  # strip namespace
+    d = {tag: {} if tree.attrib else None}
+    children = list(tree)
+    if children:
+        dd = defaultdict(list)
+        for dc in map(etree_to_dict, children):
+            for k, v in dc.items():
+                dd[k].append(v)
+        d = {tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
+    if tree.attrib:
+        d[tag].update(("@" + k, v) for k, v in tree.attrib.items())
+    if tree.text:
+        text = tree.text.strip()
+        if children or tree.attrib:
+            if text:
+                d[tag]["#text"] = text
+        else:
+            d[tag] = text
+    return d

invenio_vocabularies/ext.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2023 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -39,8 +40,14 @@ from .contrib.subjects import (
     SubjectsService,
     SubjectsServiceConfig,
 )
-from .resources.resource import VocabulariesResource
-from .services.service import VocabulariesService
+from .resources import (
+    VocabulariesAdminResource,
+    VocabulariesResource,
+    VocabulariesResourceConfig,
+    VocabularyTypeResourceConfig,
+)
+from .services.config import VocabularyTypesServiceConfig
+from .services.service import VocabulariesService, VocabularyTypeService
 
 
 class InvenioVocabularies(object):
@@ -75,6 +82,7 @@ class InvenioVocabularies(object):
             funders = FundersServiceConfig
             names = NamesServiceConfig
             subjects = SubjectsServiceConfig
+            vocabulary_types = VocabularyTypesServiceConfig
 
         return ServiceConfigs
 
@@ -92,9 +100,12 @@ class InvenioVocabularies(object):
         self.funders_service = FundersService(config=service_configs.funders)
         self.names_service = NamesService(config=service_configs.names)
         self.subjects_service = SubjectsService(config=service_configs.subjects)
-        self.service = VocabulariesService(
+        self.vocabularies_service = VocabulariesService(
             config=app.config["VOCABULARIES_SERVICE_CONFIG"],
         )
+        self.vocabulary_types_service = VocabularyTypeService(
+            config=service_configs.vocabulary_types
+        )
 
     def init_resource(self, app):
         """Initialize vocabulary resources."""
@@ -120,6 +131,49 @@ class InvenioVocabularies(object):
             config=SubjectsResourceConfig,
         )
         self.resource = VocabulariesResource(
-            service=self.service,
+            service=self.vocabularies_service,
             config=app.config["VOCABULARIES_RESOURCE_CONFIG"],
         )
+        self.vocabulary_admin_resource = VocabulariesAdminResource(
+            service=self.vocabulary_types_service,
+            config=VocabularyTypeResourceConfig,
+        )
+
+
+def finalize_app(app):
+    """Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def api_finalize_app(app):
+    """Api Finalize app.
+
+    NOTE: replace former @record_once decorator
+    """
+    init(app)
+
+
+def init(app):
+    """Init app."""
+    # Register services - cannot be done in extension because
+    # Invenio-Records-Resources might not have been initialized.
+    sregistry = app.extensions["invenio-records-resources"].registry
+    ext = app.extensions["invenio-vocabularies"]
+    sregistry.register(ext.affiliations_service, service_id="affiliations")
+    sregistry.register(ext.awards_service, service_id="awards")
+    sregistry.register(ext.funders_service, service_id="funders")
+    sregistry.register(ext.names_service, service_id="names")
+    sregistry.register(ext.subjects_service, service_id="subjects")
+    sregistry.register(ext.vocabularies_service, service_id="vocabularies")
+    sregistry.register(ext.vocabulary_types_service, service_id="vocabulary-types")
+    # Register indexers
+    iregistry = app.extensions["invenio-indexer"].registry
+    iregistry.register(ext.affiliations_service.indexer, indexer_id="affiliations")
+    iregistry.register(ext.awards_service.indexer, indexer_id="awards")
+    iregistry.register(ext.funders_service.indexer, indexer_id="funders")
+    iregistry.register(ext.names_service.indexer, indexer_id="names")
+    iregistry.register(ext.subjects_service.indexer, indexer_id="subjects")
+    iregistry.register(ext.vocabularies_service.indexer, indexer_id="vocabularies")

invenio_vocabularies/factories.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 KTH Royal Institute of Technology.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+"""Generate Vocabulary Config."""
+
+from copy import deepcopy
+
+import yaml
+from invenio_records_resources.proxies import current_service_registry
+
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG as affiliations_ds_config,
+)
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG_OPENAIRE as affiliations_openaire_ds_config,
+)
+from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
+from .contrib.awards.datastreams import (
+    DATASTREAM_CONFIG_CORDIS as awards_cordis_ds_config,
+)
+from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
+from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
+from .contrib.subjects.datastreams import DATASTREAM_CONFIG as subjects_ds_config
+
+
+class VocabularyConfig:
+    """Vocabulary Config Factory."""
+
+    config = None
+    vocabulary_name = None
+
+    def get_config(self, filepath=None, origin=None):
+        """Get the configuration for the vocabulary."""
+        config = deepcopy(self.config)
+        if filepath:
+            with open(filepath, encoding="utf-8") as f:
+                config = yaml.safe_load(f).get(self.vocabulary_name)
+        if origin:
+            config["readers"][0].setdefault("args", {})
+            config["readers"][0]["args"]["origin"] = origin
+        return config
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        return current_service_registry.get(self.vocabulary_name)
+
+
+class NamesVocabularyConfig(VocabularyConfig):
+    """Names Vocabulary Config."""
+
+    config = names_ds_config
+    vocabulary_name = "names"
+
+
+class FundersVocabularyConfig(VocabularyConfig):
+    """Funders Vocabulary Config."""
+
+    config = funders_ds_config
+    vocabulary_name = "funders"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Funders")
+
+
+class SubjectsVocabularyConfig(VocabularyConfig):
+    """Subjects Vocabulary Config."""
+
+    config = subjects_ds_config
+    vocabulary_name = "subjects"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Subjects")
+
+
+class AwardsVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_ds_config
+    vocabulary_name = "awards"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Awards")
+
+
+class AwardsCordisVocabularyConfig(VocabularyConfig):
+    """Awards Vocabulary Config."""
+
+    config = awards_cordis_ds_config
+    vocabulary_name = "awards:cordis"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for CORDIS Awards")
+
+
+class AffiliationsVocabularyConfig(VocabularyConfig):
+    """Affiliations Vocabulary Config."""
+
+    config = affiliations_ds_config
+    vocabulary_name = "affiliations"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Affiliations")
+
+
+class AffiliationsOpenAIREVocabularyConfig(VocabularyConfig):
+    """OpenAIRE Affiliations Vocabulary Config."""
+
+    config = affiliations_openaire_ds_config
+    vocabulary_name = "affiliations:openaire"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for OpenAIRE Affiliations")
+
+
+def get_vocabulary_config(vocabulary):
+    """Factory function to get the appropriate Vocabulary Config."""
+    vocab_config = {
+        "names": NamesVocabularyConfig,
+        "funders": FundersVocabularyConfig,
+        "awards": AwardsVocabularyConfig,
+        "awards:cordis": AwardsCordisVocabularyConfig,
+        "affiliations": AffiliationsVocabularyConfig,
+        "affiliations:openaire": AffiliationsOpenAIREVocabularyConfig,
+        "subjects": SubjectsVocabularyConfig,
+    }
+    return vocab_config.get(vocabulary, VocabularyConfig)()

invenio_vocabularies/jobs.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Jobs module."""
+
+import datetime
+from datetime import timezone
+
+from invenio_i18n import gettext as _
+from invenio_jobs.jobs import JobType
+from marshmallow import Schema, fields
+from marshmallow_utils.fields import TZDateTime
+
+from invenio_vocabularies.services.tasks import process_datastream
+
+
+class ArgsSchema(Schema):
+    """Schema of task input arguments."""
+
+    since = TZDateTime(
+        timezone=timezone.utc,
+        format="iso",
+        metadata={
+            "description": _(
+                "YYYY-MM-DD HH:mm format. "
+                "Leave field empty if it should continue since last successful run."
+            )
+        },
+    )
+    job_arg_schema = fields.String(
+        metadata={"type": "hidden"},
+        dump_default="ArgsSchema",
+        load_default="ArgsSchema",
+    )
+
+
+class ProcessDataStreamJob(JobType):
+    """Generic process data stream job type."""
+
+    arguments_schema = ArgsSchema
+    task = process_datastream
+    id = None
+
+
+class ProcessRORAffiliationsJob(ProcessDataStreamJob):
+    """Process ROR affiliations datastream registered task."""
+
+    description = "Process ROR affiliations"
+    title = "Load ROR affiliations"
+    id = "process_ror_affiliations"
+
+    @classmethod
+    def default_args(cls, job_obj, since=None, **kwargs):
+        """Generate default job arguments here."""
+        if since is None and job_obj.last_runs["success"]:
+            since = job_obj.last_runs["success"].started_at
+        else:
+            since = since or datetime.datetime.now()
+
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": str(since)},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "_schema_v2\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "affiliations-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-affiliations"}],
+            }
+        }
+
+
+class ProcessRORFundersJob(ProcessDataStreamJob):
+    """Process ROR funders datastream registered task."""
+
+    description = "Process ROR funders"
+    title = "Load ROR funders"
+    id = "process_ror_funders"
+
+    @classmethod
+    def default_args(cls, job_obj, since=None, **kwargs):
+        """Generate default job arguments here."""
+        if since is None and job_obj.last_runs["success"]:
+            since = job_obj.last_runs["success"].started_at
+        else:
+            since = since or datetime.datetime.now()
+
+        # NOTE: Update is set to False for now given we don't have the logic to re-index dependent records yet.
+        # Since jobs support custom args, update true can be passed via that.
+        return {
+            "config": {
+                "readers": [
+                    {
+                        "args": {"since": str(since)},
+                        "type": "ror-http",
+                    },
+                    {"args": {"regex": "_schema_v2\\.json$"}, "type": "zip"},
+                    {"type": "json"},
+                ],
+                "writers": [
+                    {
+                        "args": {
+                            "writer": {
+                                "type": "funders-service",
+                                "args": {"update": False},
+                            }
+                        },
+                        "type": "async",
+                    }
+                ],
+                "transformers": [{"type": "ror-funders"}],
+            }
+        }

invenio_vocabularies/proxies.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -19,7 +19,7 @@ def _ext_proxy(attr):
     )
 
 
-current_service = _ext_proxy("service")
+current_service = _ext_proxy("vocabularies_service")
 """Proxy to the instantiated vocabulary service."""
 
 

invenio_vocabularies/records/jsonschemas/vocabularies/definitions-v1.0.0.json

@@ -10,5 +10,12 @@
   },
   "icon": {
     "type": "string"
+  },
+  "tags": {
+    "type": "array",
+    "description": "Tags for a vocabulary item.",
+    "items": {
+      "type": "string"
+    }
   }
 }

invenio_vocabularies/records/jsonschemas/vocabularies/vocabulary-v1.0.0.json

@@ -34,10 +34,7 @@
       }
     },
     "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
     },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"

invenio_vocabularies/records/mappings/os-v1/vocabularies/vocabulary-v1.0.0.json

@@ -81,7 +81,7 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true,
+        "dynamic": "true",
         "properties": {
           "en": {
             "type": "search_as_you_type",
@@ -91,7 +91,7 @@
       },
       "description": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       },
       "icon": {
         "type": "keyword",
@@ -102,7 +102,7 @@
       },
       "props": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       }
     }
   }

invenio_vocabularies/records/mappings/os-v2/vocabularies/vocabulary-v1.0.0.json

@@ -81,7 +81,7 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true,
+        "dynamic": "true",
         "properties": {
           "en": {
             "type": "search_as_you_type",
@@ -91,7 +91,7 @@
       },
       "description": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       },
       "icon": {
         "type": "keyword",
@@ -102,7 +102,7 @@
       },
       "props": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       }
     }
   }

invenio_vocabularies/records/mappings/v7/vocabularies/vocabulary-v1.0.0.json

@@ -81,7 +81,7 @@
       },
       "title": {
         "type": "object",
-        "dynamic": true,
+        "dynamic": "true",
         "properties": {
           "en": {
             "type": "search_as_you_type",
@@ -91,7 +91,7 @@
       },
       "description": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       },
       "icon": {
         "type": "keyword",
@@ -102,7 +102,7 @@
       },
       "props": {
         "type": "object",
-        "dynamic": true
+        "dynamic": "true"
       }
     }
   }

invenio_vocabularies/records/models.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -79,9 +79,7 @@ class VocabularyScheme(db.Model):
         """Create a new vocabulary subtype."""
         banned = [",", ":"]
         for b in banned:
-            assert (
-                b not in data["id"]
-            ), f"No '{b}' allowed in VocabularyScheme.id"  # noqa
+            assert b not in data["id"], f"No '{b}' allowed in VocabularyScheme.id"
 
         with db.session.begin_nested():
             obj = cls(**data)

invenio_vocabularies/records/pidprovider.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Persistent identifier provider for vocabularies."""
 
-
 from invenio_pidstore.models import PIDStatus
 from invenio_pidstore.providers.base import BaseProvider
 

invenio_vocabularies/records/systemfields/relations.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2024 CERN.
 #
 # Invenio-Records-Resources is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -38,7 +38,7 @@ class CustomFieldsRelation(RelationsField):
                 relations[cf.name] = cf.relation_cls(
                     f"custom_fields.{cf.name}",
                     keys=cf.field_keys,
-                    pid_field=Vocabulary.pid.with_type_ctx(cf.vocabulary_id),
+                    pid_field=cf.pid_field,
                     cache_key=cf.vocabulary_id,
                 )
 

invenio_vocabularies/resources/__init__.py

@@ -1,15 +1,23 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Resources module."""
+
 from invenio_vocabularies.resources.schema import L10NString, VocabularyL10Schema
 
+from .config import VocabulariesResourceConfig, VocabularyTypeResourceConfig
+from .resource import VocabulariesAdminResource, VocabulariesResource
+
 __all__ = (
     "VocabularyL10Schema",
     "L10NString",
+    "VocabulariesResourceConfig",
+    "VocabularyTypeResourceConfig",
+    "VocabulariesAdminResource",
+    "VocabulariesResource",
 )