invenio-vocabularies 4.0.0__py2.py3-none-any.whl → 4.2.0__py2.py3-none-any.whl

invenio_vocabularies/datastreams/readers.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021-2024 CERN.
+# Copyright (C)      2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -19,11 +20,17 @@ from json.decoder import JSONDecodeError
 
 import requests
 import yaml
+from lxml import etree
 from lxml.html import parse as html_parse
 
 from .errors import ReaderError
 from .xml import etree_to_dict
 
+try:
+    import oaipmh_scythe
+except ImportError:
+    oaipmh_scythe = None
+
 
 class BaseReader(ABC):
     """Base reader."""
@@ -226,3 +233,99 @@ class XMLReader(BaseReader):
             raise ReaderError(f"Record not found in XML entry.")
 
         yield record
+
+
+class OAIPMHReader(BaseReader):
+    """OAIPMH reader."""
+
+    def __init__(
+        self,
+        *args,
+        base_url=None,
+        metadata_prefix=None,
+        set=None,
+        from_date=None,
+        until_date=None,
+        verb=None,
+        **kwargs,
+    ):
+        """Constructor."""
+        self._base_url = base_url
+        self._metadata_prefix = metadata_prefix if not None else "oai_dc"
+        self._set = set
+        self._until = until_date
+        self._from = from_date
+        self._verb = verb if not None else "ListRecords"
+        super().__init__(*args, **kwargs)
+
+    def _iter(self, scythe, *args, **kwargs):
+        """Read and parse an OAIPMH stream to dict."""
+
+        class OAIRecord(oaipmh_scythe.models.Record):
+            """An XML unpacking implementation for more complicated formats."""
+
+            def get_metadata(self):
+                """Extract and return the record's metadata as a dictionary."""
+                return xml_to_dict(
+                    self.xml.find(f".//{self._oai_namespace}metadata").getchildren()[0],
+                )
+
+        if self._verb == "ListRecords":
+            scythe.class_mapping["ListRecords"] = OAIRecord
+            try:
+                records = scythe.list_records(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for record in records:
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+        else:
+            scythe.class_mapping["GetRecord"] = OAIRecord
+            try:
+                headers = scythe.list_identifiers(
+                    from_=self._from,
+                    until=self._until,
+                    metadata_prefix=self._metadata_prefix,
+                    set_=self._set,
+                    ignore_deleted=True,
+                )
+                for header in headers:
+                    record = scythe.get_record(
+                        identifier=header.identifier,
+                        metadata_prefix=self._metadata_prefix,
+                    )
+                    yield {"record": record}
+            except oaipmh_scythe.NoRecordsMatch:
+                raise ReaderError("No records found in OAI-PMH request.")
+
+    def read(self, item=None, *args, **kwargs):
+        """Reads from item or opens the file descriptor from origin."""
+        if item:
+            raise NotImplementedError(
+                "OAIPMHReader does not support being chained after another reader"
+            )
+        else:
+            with oaipmh_scythe.Scythe(self._base_url) as scythe:
+                yield from self._iter(scythe=scythe, *args, **kwargs)
+
+
+def xml_to_dict(tree: etree._Element):
+    """Convert an XML tree to a dictionary.
+
+    This function takes an XML element tree and converts it into a dictionary.
+
+    Args:
+        tree: The root element of the XML tree to be converted.
+
+    Returns:
+        A dictionary with the key "record".
+    """
+    dict_obj = dict()
+    dict_obj["record"] = etree.tostring(tree)
+
+    return dict_obj

invenio_vocabularies/datastreams/tasks.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022-2024 CERN.
+#
+# Invenio-Vocabularies is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Data Streams Celery tasks."""
+
+from celery import shared_task
+
+from ..datastreams import StreamEntry
+from ..datastreams.factories import WriterFactory
+
+
+@shared_task(ignore_result=True)
+def write_entry(writer, entry):
+    """Write an entry.
+
+    :param writer: writer configuration as accepted by the WriterFactory.
+    :param entry: dictionary, StreamEntry is not serializable.
+    """
+    writer = WriterFactory.create(config=writer)
+    writer.write(StreamEntry(entry))

invenio_vocabularies/datastreams/writers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021-2022 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -20,6 +20,7 @@ from marshmallow import ValidationError
 
 from .datastreams import StreamEntry
 from .errors import WriterError
+from .tasks import write_entry
 
 
 class BaseWriter(ABC):
@@ -103,6 +104,24 @@ class YamlWriter(BaseWriter):
         with open(self._filepath, "a") as file:
             # made into array for safer append
             # will always read array (good for reader)
-            yaml.safe_dump([stream_entry.entry], file)
+            yaml.safe_dump([stream_entry.entry], file, allow_unicode=True)
+
+        return stream_entry
+
+
+class AsyncWriter(BaseWriter):
+    """Writes the entries asynchronously (celery task)."""
+
+    def __init__(self, writer, *args, **kwargs):
+        """Constructor.
+
+        :param writer: writer to use.
+        """
+        self._writer = writer
+        super().__init__(*args, **kwargs)
+
+    def write(self, stream_entry, *args, **kwargs):
+        """Launches a celery task to write an entry."""
+        write_entry.delay(self._writer, stream_entry.entry)
 
         return stream_entry

invenio_vocabularies/ext.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2023 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -40,8 +40,14 @@ from .contrib.subjects import (
     SubjectsService,
     SubjectsServiceConfig,
 )
-from .resources.resource import VocabulariesResource
-from .services.service import VocabulariesService
+from .resources import (
+    VocabulariesAdminResource,
+    VocabulariesResource,
+    VocabulariesResourceConfig,
+    VocabularyTypeResourceConfig,
+)
+from .services.config import VocabularyTypesServiceConfig
+from .services.service import VocabulariesService, VocabularyTypeService
 
 
 class InvenioVocabularies(object):
@@ -76,6 +82,7 @@ class InvenioVocabularies(object):
             funders = FundersServiceConfig
             names = NamesServiceConfig
             subjects = SubjectsServiceConfig
+            vocabulary_types = VocabularyTypesServiceConfig
 
         return ServiceConfigs
 
@@ -93,9 +100,12 @@ class InvenioVocabularies(object):
         self.funders_service = FundersService(config=service_configs.funders)
         self.names_service = NamesService(config=service_configs.names)
         self.subjects_service = SubjectsService(config=service_configs.subjects)
-        self.service = VocabulariesService(
+        self.vocabularies_service = VocabulariesService(
             config=app.config["VOCABULARIES_SERVICE_CONFIG"],
         )
+        self.vocabulary_types_service = VocabularyTypeService(
+            config=service_configs.vocabulary_types
+        )
 
     def init_resource(self, app):
         """Initialize vocabulary resources."""
@@ -121,9 +131,13 @@ class InvenioVocabularies(object):
             config=SubjectsResourceConfig,
         )
         self.resource = VocabulariesResource(
-            service=self.service,
+            service=self.vocabularies_service,
             config=app.config["VOCABULARIES_RESOURCE_CONFIG"],
         )
+        self.vocabulary_admin_resource = VocabulariesAdminResource(
+            service=self.vocabulary_types_service,
+            config=VocabularyTypeResourceConfig,
+        )
 
 
 def finalize_app(app):
@@ -153,7 +167,8 @@ def init(app):
     sregistry.register(ext.funders_service, service_id="funders")
     sregistry.register(ext.names_service, service_id="names")
     sregistry.register(ext.subjects_service, service_id="subjects")
-    sregistry.register(ext.service, service_id="vocabularies")
+    sregistry.register(ext.vocabularies_service, service_id="vocabularies")
+    sregistry.register(ext.vocabulary_types_service, service_id="vocabulary-types")
     # Register indexers
     iregistry = app.extensions["invenio-indexer"].registry
     iregistry.register(ext.affiliations_service.indexer, indexer_id="affiliations")
@@ -161,4 +176,4 @@ def init(app):
     iregistry.register(ext.funders_service.indexer, indexer_id="funders")
     iregistry.register(ext.names_service.indexer, indexer_id="names")
     iregistry.register(ext.subjects_service.indexer, indexer_id="subjects")
-    iregistry.register(ext.service.indexer, indexer_id="vocabularies")
+    iregistry.register(ext.vocabularies_service.indexer, indexer_id="vocabularies")

invenio_vocabularies/factories.py

@@ -7,11 +7,15 @@
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 """Generate Vocabulary Config."""
+
 from copy import deepcopy
 
 import yaml
 from invenio_records_resources.proxies import current_service_registry
 
+from .contrib.affiliations.datastreams import (
+    DATASTREAM_CONFIG as affiliations_ds_config,
+)
 from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
 from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
 from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
@@ -68,11 +72,23 @@ class AwardsVocabularyConfig(VocabularyConfig):
         raise NotImplementedError("Service not implemented for Awards")
 
 
+class AffiliationsVocabularyConfig(VocabularyConfig):
+    """Affiliations Vocabulary Config."""
+
+    config = affiliations_ds_config
+    vocabulary_name = "affiliations"
+
+    def get_service(self):
+        """Get the service for the vocabulary."""
+        raise NotImplementedError("Service not implemented for Affiliations")
+
+
 def get_vocabulary_config(vocabulary):
     """Factory function to get the appropriate Vocabulary Config."""
     vocab_config = {
         "names": NamesVocabularyConfig,
         "funders": FundersVocabularyConfig,
         "awards": AwardsVocabularyConfig,
+        "affiliations": AffiliationsVocabularyConfig,
     }
     return vocab_config.get(vocabulary, VocabularyConfig)()

invenio_vocabularies/proxies.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -19,7 +19,7 @@ def _ext_proxy(attr):
     )
 
 
-current_service = _ext_proxy("service")
+current_service = _ext_proxy("vocabularies_service")
 """Proxy to the instantiated vocabulary service."""
 
 

invenio_vocabularies/records/jsonschemas/vocabularies/definitions-v1.0.0.json

@@ -10,5 +10,12 @@
   },
   "icon": {
     "type": "string"
+  },
+  "tags": {
+    "type": "array",
+    "description": "Tags for a vocabulary item.",
+    "items": {
+      "type": "string"
+    }
   }
 }

invenio_vocabularies/records/jsonschemas/vocabularies/vocabulary-v1.0.0.json

@@ -34,10 +34,7 @@
       }
     },
     "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
+      "$ref": "local://vocabularies/definitions-v1.0.0.json#/tags"
     },
     "title": {
       "$ref": "local://vocabularies/definitions-v1.0.0.json#/title"

invenio_vocabularies/records/models.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -79,9 +79,7 @@ class VocabularyScheme(db.Model):
         """Create a new vocabulary subtype."""
         banned = [",", ":"]
         for b in banned:
-            assert (
-                b not in data["id"]
-            ), f"No '{b}' allowed in VocabularyScheme.id"  # noqa
+            assert b not in data["id"], f"No '{b}' allowed in VocabularyScheme.id"
 
         with db.session.begin_nested():
             obj = cls(**data)

invenio_vocabularies/records/pidprovider.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Persistent identifier provider for vocabularies."""
 
-
 from invenio_pidstore.models import PIDStatus
 from invenio_pidstore.providers.base import BaseProvider
 

invenio_vocabularies/resources/__init__.py

@@ -1,15 +1,23 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
 """Resources module."""
+
 from invenio_vocabularies.resources.schema import L10NString, VocabularyL10Schema
 
+from .config import VocabulariesResourceConfig, VocabularyTypeResourceConfig
+from .resource import VocabulariesAdminResource, VocabulariesResource
+
 __all__ = (
     "VocabularyL10Schema",
     "L10NString",
+    "VocabulariesResourceConfig",
+    "VocabularyTypeResourceConfig",
+    "VocabulariesAdminResource",
+    "VocabulariesResource",
 )

invenio_vocabularies/resources/config.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2024 University of Münster.
+#
+# Invenio-Vocabularies is free software; you can redistringibute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Resources config."""
+
+import marshmallow as ma
+from flask_resources import (
+    BaseListSchema,
+    HTTPJSONException,
+    JSONSerializer,
+    MarshmallowSerializer,
+    ResourceConfig,
+    ResponseHandler,
+    create_error_handler,
+)
+from invenio_records_resources.resources import (
+    RecordResource,
+    RecordResourceConfig,
+    SearchRequestArgsSchema,
+)
+from invenio_records_resources.resources.errors import ErrorHandlersMixin
+from invenio_records_resources.resources.records.args import SearchRequestArgsSchema
+from invenio_records_resources.resources.records.headers import etag_headers
+from invenio_records_resources.services.base.config import ConfiguratorMixin
+
+from .serializer import VocabularyL10NItemSchema
+
+
+class VocabularySearchRequestArgsSchema(SearchRequestArgsSchema):
+    """Vocabularies search request parameters."""
+
+    tags = ma.fields.Str()
+    active = ma.fields.Boolean()
+    status = ma.fields.Boolean()
+
+
+class VocabulariesResourceConfig(RecordResourceConfig):
+    """Vocabulary resource configuration."""
+
+    blueprint_name = "vocabularies"
+    url_prefix = "/vocabularies"
+    routes = {
+        "list": "/<type>",
+        "item": "/<type>/<pid_value>",
+        "tasks": "/tasks",
+    }
+
+    request_view_args = {
+        "pid_value": ma.fields.Str(),
+        "type": ma.fields.Str(required=True),
+    }
+
+    request_search_args = VocabularySearchRequestArgsSchema
+
+    response_handlers = {
+        "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+        "application/vnd.inveniordm.v1+json": ResponseHandler(
+            MarshmallowSerializer(
+                format_serializer_cls=JSONSerializer,
+                object_schema_cls=VocabularyL10NItemSchema,
+                list_schema_cls=BaseListSchema,
+            ),
+            headers=etag_headers,
+        ),
+    }
+
+
+class VocabularyTypeResourceConfig(ResourceConfig, ConfiguratorMixin):
+    """Vocabulary list resource config."""
+
+    # /vocabulary/vocabulary_id
+    # Blueprint configuration
+    blueprint_name = "vocabulary_types"
+    url_prefix = "/vocabularies"
+
+    routes = {"list": "/"}
+
+    # Request parsing
+    request_read_args = {}
+    request_view_args = {
+        "pid_value": ma.fields.String,
+        "type": ma.fields.String,
+    }
+    request_search_args = VocabularySearchRequestArgsSchema
+
+    error_handlers = {
+        **ErrorHandlersMixin.error_handlers,
+        # TODO: Add custom error handlers here
+    }
+    response_handlers = {
+        "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+        "application/vnd.inveniordm.v1+json": ResponseHandler(
+            MarshmallowSerializer(
+                format_serializer_cls=JSONSerializer,
+                object_schema_cls=VocabularyL10NItemSchema,
+                list_schema_cls=BaseListSchema,
+            ),
+            headers=etag_headers,
+        ),
+    }

invenio_vocabularies/resources/resource.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
+# Copyright (C) 2024 Uni Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,6 +9,8 @@
 
 """Vocabulary resource."""
 
+import json
+
 import marshmallow as ma
 from flask import g
 from flask_resources import (
@@ -18,6 +21,7 @@ from flask_resources import (
     resource_requestctx,
     response_handler,
 )
+from invenio_access.permissions import system_identity
 from invenio_records_resources.resources import (
     RecordResource,
     RecordResourceConfig,
@@ -37,55 +41,20 @@ from marshmallow import fields
 from .serializer import VocabularyL10NItemSchema
 
 
-#
-# Request args
-#
-class VocabularySearchRequestArgsSchema(SearchRequestArgsSchema):
-    """Add parameter to parse tags."""
-
-    tags = fields.Str()
-
-
-#
-# Resource config
-#
-class VocabulariesResourceConfig(RecordResourceConfig):
-    """Vocabulary resource configuration."""
-
-    blueprint_name = "vocabularies"
-    url_prefix = "/vocabularies"
-    routes = {"list": "/<type>", "item": "/<type>/<pid_value>", "tasks": "/tasks"}
-
-    request_view_args = {
-        "pid_value": ma.fields.Str(),
-        "type": ma.fields.Str(required=True),
-    }
-
-    request_search_args = VocabularySearchRequestArgsSchema
-
-    response_handlers = {
-        "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
-        "application/vnd.inveniordm.v1+json": ResponseHandler(
-            MarshmallowSerializer(
-                format_serializer_cls=JSONSerializer,
-                object_schema_cls=VocabularyL10NItemSchema,
-                list_schema_cls=BaseListSchema,
-            ),
-            headers=etag_headers,
-        ),
-    }
-
-
 #
 # Resource
 #
 class VocabulariesResource(RecordResource):
-    """Resource for generic vocabularies."""
+    """Resource for generic vocabularies.
+
+    Provide the API /api/vocabularies/
+    """
 
     def create_url_rules(self):
         """Create the URL rules for the record resource."""
         routes = self.config.routes
         rules = super().create_url_rules()
+
         rules.append(
             route("POST", routes["tasks"], self.launch),
         )
@@ -164,3 +133,24 @@ class VocabulariesResource(RecordResource):
         """Create a task."""
         self.service.launch(g.identity, resource_requestctx.data or {})
         return "", 202
+
+
+class VocabulariesAdminResource(RecordResource):
+    """Resource for vocabularies admin interface."""
+
+    def create_url_rules(self):
+        """Create the URL rules for the record resource."""
+        routes = self.config.routes
+
+        rules = [route("GET", routes["list"], self.search)]
+
+        return rules
+
+    @request_search_args
+    @response_handler(many=True)
+    def search(self):
+        """Return information about _all_ vocabularies."""
+        identity = g.identity
+        hits = self.service.search(identity, params=resource_requestctx.args)
+
+        return hits.to_dict(), 200

invenio_vocabularies/resources/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -8,6 +8,7 @@
 # details.
 
 """Vocabulary resource schema."""
+
 from marshmallow import Schema, fields
 
 from invenio_vocabularies.resources.serializer import L10NString

invenio_vocabularies/services/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,9 +8,12 @@
 
 """Services module."""
 
-from .service import VocabulariesService, VocabulariesServiceConfig
+from .config import VocabulariesServiceConfig, VocabularyTypesServiceConfig
+from .service import VocabulariesService, VocabularyTypeService
 
 __all__ = (
     "VocabulariesService",
+    "VocabularyTypeService",
     "VocabulariesServiceConfig",
+    "VocabularyTypesServiceConfig",
 )