invenio-vocabularies 1.2.0__py2.py3-none-any.whl → 6.3.1__py2.py3-none-any.whl

invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/contrib/index.js

@@ -0,0 +1,7 @@
+// This file is part of InvenioVocabularies
+// Copyright (C) 2023 CERN.
+//
+// Invenio is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+export * from "./forms";

invenio_vocabularies/assets/semantic-ui/js/invenio_vocabularies/src/index.js

@@ -0,0 +1,7 @@
+// This file is part of InvenioVocabularies
+// Copyright (C) 2023 CERN.
+//
+// Invenio is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+export * from "./contrib";

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/i18next-scanner.config.js

@@ -0,0 +1,63 @@
+// This file is part of React-Invenio-Deposit
+//
+// Invenio-administration is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+// list of func used to
+// mark the strings for translation
+const { languages } = require("./package.json").config;
+
+const funcList = ["i18next.t"];
+const extensions = [".js", ".jsx"];
+
+module.exports = {
+  options: {
+    debug: true,
+    removeUnusedKeys: true,
+    browserLanguageDetection: true,
+    func: {
+      list: funcList,
+      extensions: extensions,
+    },
+    //using Trans component
+    trans: {
+      component: "Trans",
+      extensions: extensions,
+      fallbackKey: function (ns, value) {
+        return value;
+      },
+    },
+    lngs: languages,
+    ns: [
+      // file name (.json)
+      "translations",
+    ],
+    defaultLng: "en",
+    defaultNs: "translations",
+    // @param {string} lng The language currently used.
+    // @param {string} ns The namespace currently used.
+    // @param {string} key The translation key.
+    // @return {string} Returns a default value for the translation key.
+    defaultValue: function (lng, ns, key) {
+      if (lng === "en") {
+        // Return key as the default value for English language
+        return key;
+      }
+      return "";
+    },
+    resource: {
+      // The path where resources get loaded from. Relative to current working directory.
+      loadPath: "messages/{{lng}}/{{ns}}.json",
+
+      // The path to store resources.
+      savePath: "messages/{{lng}}/{{ns}}.json",
+      jsonIndent: 2,
+      lineEnding: "\n",
+    },
+    nsSeparator: false, // namespace separator
+
+    //Set to false to disable key separator
+    // if you prefer having keys as the fallback for translation (e.g. gettext).
+    keySeparator: false,
+  },
+};

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/i18next.js

@@ -0,0 +1,36 @@
+// This file is part of React-Invenio-Deposit
+//
+// Invenio-administration is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+import i18n from "i18next";
+
+import LanguageDetector from "i18next-browser-languagedetector";
+import { translations } from "./messages";
+import { initReactI18next } from "react-i18next";
+
+const options = {
+  fallbackLng: "en", // fallback keys
+  returnEmptyString: false,
+  debug: process.env.NODE_ENV === "development",
+  resources: translations,
+  keySeparator: false,
+  nsSeparator: false,
+  // specify language detection order
+  detection: {
+    order: ["htmlTag"],
+    // cache user language off
+    caches: [],
+  },
+  react: {
+    // Set empty - to allow html tags convert to trans tags
+    // HTML TAG | Trans TAG
+    //  <span>  | <1>
+    transKeepBasicHtmlNodesFor: [],
+  },
+};
+
+const i18next = i18n.createInstance();
+i18next.use(LanguageDetector).use(initReactI18next).init(options);
+
+export { i18next };

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/messages/index.js

@@ -0,0 +1 @@
+export const translations = {};

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "invenio-vocabularies-ui",
+  "config": {
+    "languages": [
+      "af",
+      "ar",
+      "bg",
+      "ca",
+      "cs",
+      "da",
+      "de",
+      "el",
+      "en",
+      "es",
+      "et",
+      "et_EE",
+      "fa",
+      "fr",
+      "gl",
+      "hr",
+      "hu",
+      "it",
+      "ja",
+      "ka",
+      "lt",
+      "no",
+      "pl",
+      "pt",
+      "ro",
+      "ru",
+      "rw",
+      "sk",
+      "sv",
+      "tr",
+      "uk",
+      "zh_CN",
+      "zh_TW"
+    ]
+  },
+  "devDependencies": {
+    "i18next-conv": "^10.2.0",
+    "i18next-scanner": "^3.0.0",
+    "react-i18next": "^11.11.3",
+    "i18next": "^20.3.0",
+    "i18next-browser-languagedetector": "^6.1.0"
+  },
+  "scripts": {
+    "extract_messages": "i18next-scanner --config i18next-scanner.config.js '../../js/**/*.{js,jsx}'",
+    "postextract_messages": "i18next-conv -l en -s ./messages/en/translations.json -t ./translations.pot",
+    "compile_catalog": "node ./scripts/compileCatalog.js",
+    "init_catalog": "node ./scripts/initCatalog"
+  }
+}

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/scripts/compileCatalog.js

@@ -0,0 +1,39 @@
+// This file is part of React-Invenio-Deposit
+//
+// Invenio-administration is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+const { readFileSync, writeFileSync } = require("fs");
+const { gettextToI18next } = require("i18next-conv");
+
+const PACKAGE_JSON_BASE_PATH = "./";
+const { languages } = require(`../package`).config;
+
+// it accepts the same options as the cli.
+// https://github.com/i18next/i18next-gettext-converter#options
+const options = {
+  /* you options here */
+};
+
+function save(target) {
+  return (result) => {
+    writeFileSync(target, result);
+  };
+}
+
+if ("lang" === process.argv[2]) {
+  const lang = process.argv[3];
+  gettextToI18next(
+    lang,
+    readFileSync(`${PACKAGE_JSON_BASE_PATH}messages/${lang}/messages.po`),
+    options
+  ).then(save(`${PACKAGE_JSON_BASE_PATH}messages/${lang}/translations.json`));
+} else {
+  for (const lang of languages) {
+    gettextToI18next(
+      lang,
+      readFileSync(`${PACKAGE_JSON_BASE_PATH}messages/${lang}/messages.po`),
+      options
+    ).then(save(`${PACKAGE_JSON_BASE_PATH}messages/${lang}/translations.json`));
+  }
+}

invenio_vocabularies/assets/semantic-ui/translations/invenio_vocabularies/scripts/initCatalog.js

@@ -0,0 +1,19 @@
+// This file is part of React-Invenio-Deposit
+//
+// Invenio-administration is free software; you can redistribute it and/or modify it
+// under the terms of the MIT License; see LICENSE file for more details.
+
+const { writeFileSync } = require("fs");
+const packageJson = require("../package");
+
+const { languages } = packageJson.config;
+if ("lang" === process.argv[2]) {
+  const addedLang = process.argv[3];
+  languages.push(addedLang);
+  packageJson.config.languages = [...new Set(languages)];
+  writeFileSync(`package.json`, JSON.stringify(packageJson, null, 2));
+} else {
+  console.error(
+    "Error:Please provide a language by running `npm run init_catalog lang <lang>`"
+  );
+}

invenio_vocabularies/cli.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Graz University of Technology.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
@@ -9,46 +9,13 @@
 
 """Commands to create and manage vocabularies."""
 
-from copy import deepcopy
-
 import click
-import yaml
 from flask.cli import with_appcontext
 from invenio_access.permissions import system_identity
 from invenio_pidstore.errors import PIDDeletedError, PIDDoesNotExistError
-from invenio_records_resources.proxies import current_service_registry
 
-from .contrib.awards.datastreams import DATASTREAM_CONFIG as awards_ds_config
-from .contrib.funders.datastreams import DATASTREAM_CONFIG as funders_ds_config
-from .contrib.names.datastreams import DATASTREAM_CONFIG as names_ds_config
 from .datastreams import DataStreamFactory
-
-
-def get_config_for_ds(vocabulary, filepath=None, origin=None):
-    """Calculates the configuration for a Data Stream."""
-    config = None
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        config = deepcopy(names_ds_config)
-    elif vocabulary == "funders":
-        config = deepcopy(funders_ds_config)
-    elif vocabulary == "awards":
-        config = deepcopy(awards_ds_config)
-
-    if config:
-        if filepath:
-            with open(filepath) as f:
-                config = yaml.safe_load(f).get(vocabulary)
-        if origin:
-            config["readers"][0].setdefault("args", {})
-            config["readers"][0]["args"]["origin"] = origin
-
-    return config
-
-
-def get_service_for_vocabulary(vocabulary):
-    """Calculates the configuration for a Data Stream."""
-    if vocabulary == "names":  # FIXME: turn into a proper factory
-        return current_service_registry.get("names")
+from .factories import get_vocabulary_config
 
 
 @click.group()
@@ -111,7 +78,9 @@ def import_vocab(vocabulary, filepath=None, origin=None, num_samples=None):
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
+
     success, errored, filtered = _process_vocab(config, num_samples)
 
     _output_process(vocabulary, "imported", success, errored, filtered)
@@ -127,11 +96,19 @@ def update(vocabulary, filepath=None, origin=None):
     if not filepath and not origin:
         click.secho("One of --filepath or --origin must be present.", fg="red")
         exit(1)
-
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
 
     for w_conf in config["writers"]:
-        w_conf["args"]["update"] = True
+        if w_conf["type"] == "async":
+            w_conf_update = w_conf["args"]["writer"]
+        else:
+            w_conf_update = w_conf
+
+        if "args" in w_conf_update:
+            w_conf_update["args"]["update"] = True
+        else:
+            w_conf_update["args"] = {"update": True}
 
     success, errored, filtered = _process_vocab(config)
 
@@ -153,7 +130,8 @@ def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=Non
         )
         exit(1)
 
-    config = get_config_for_ds(vocabulary, filepath, origin)
+    vc = get_vocabulary_config(vocabulary)
+    config = vc.get_config(filepath, origin)
     if not filepath:
         config["writers"] = [{"type": "yaml", "args": {"filepath": target}}]
 
@@ -169,18 +147,27 @@ def convert(vocabulary, filepath=None, origin=None, target=None, num_samples=Non
     type=click.STRING,
     help="Identifier of the vocabulary item to delete.",
 )
-@click.option("--all", is_flag=True, default=False, help="Not supported yet.")
+@click.option("--all", is_flag=True, default=False)
 @with_appcontext
 def delete(vocabulary, identifier, all):
     """Delete all items or a specific one of the vocabulary."""
-    if not id and not all:
+    if not identifier and not all:
         click.secho("An identifier or the --all flag must be present.", fg="red")
         exit(1)
 
-    service = get_service_for_vocabulary(vocabulary)
+    vc = get_vocabulary_config(vocabulary)
+    service = vc.get_service()
     if identifier:
         try:
-            if service.delete(identifier, system_identity):
+            if service.delete(system_identity, identifier):
                 click.secho(f"{identifier} deleted from {vocabulary}.", fg="green")
         except (PIDDeletedError, PIDDoesNotExistError):
             click.secho(f"PID {identifier} not found.")
+    elif all:
+        items = service.scan(system_identity)
+        for item in items.hits:
+            try:
+                if service.delete(system_identity, item["id"]):
+                    click.secho(f"{item['id']} deleted from {vocabulary}.", fg="green")
+            except (PIDDeletedError, PIDDoesNotExistError):
+                click.secho(f"PID {item['id']} not found.")

invenio_vocabularies/config.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 Northwestern University.
+# Copyright (C) 2024 University of Münster.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -17,15 +18,16 @@ from .datastreams.readers import (
     GzipReader,
     JsonLinesReader,
     JsonReader,
+    OAIPMHReader,
     TarReader,
     XMLReader,
     YamlReader,
     ZipReader,
 )
 from .datastreams.transformers import XMLTransformer
-from .datastreams.writers import ServiceWriter, YamlWriter
-from .resources.resource import VocabulariesResourceConfig
-from .services.service import VocabulariesServiceConfig
+from .datastreams.writers import AsyncWriter, ServiceWriter, YamlWriter
+from .resources import VocabulariesResourceConfig
+from .services.config import VocabulariesServiceConfig
 
 VOCABULARIES_RESOURCE_CONFIG = VocabulariesResourceConfig
 """Configure the resource."""
@@ -41,8 +43,17 @@ VOCABULARIES_IDENTIFIER_SCHEMES = {
 }
 """"Generic identifier schemes, usable by other vocabularies."""
 
+
+def is_pic(val):
+    """Test if argument is a Participant Identification Code (PIC)."""
+    if len(val) != 9:
+        return False
+    return val.isdigit()
+
+
 VOCABULARIES_AFFILIATION_SCHEMES = {
     **VOCABULARIES_IDENTIFIER_SCHEMES,
+    "pic": {"label": _("PIC"), "validator": is_pic},
 }
 """Affiliations allowed identifier schemes."""
 
@@ -62,28 +73,33 @@ VOCABULARIES_AWARD_SCHEMES = {
 """Awards allowed identifier schemes."""
 
 VOCABULARIES_AWARDS_OPENAIRE_FUNDERS = {
-    "anr_________": "00rbzpz17",
     "aka_________": "05k73zm37",
+    "anr_________": "00rbzpz17",
     "arc_________": "05mmh0f86",
+    "asap________": "03zj4c476",
     "cihr________": "01gavpb45",
     "corda_______": "00k4n6c32",
+    "corda_____he": "00k4n6c32",
     "corda__h2020": "00k4n6c32",
     "euenvagency_": "02k4b9v70",
     "fct_________": "00snfqn58",
     "fwf_________": "013tf3c58",
+    "inca________": "03m8vkq32",
     "irb_hr______": "03n51vw80",
+    "lcs_________": "02ar66p97",
     "mestd_______": "01znas443",
     "nhmrc_______": "011kf5r70",
     "nih_________": "01cwqze88",
     "nserc_______": "01h531d29",
     "nsf_________": "021nxhr62",
     "nwo_________": "04jsz6e67",
-    "rcuk________": "10.13039/501100000690",
-    "ukri________": "001aqnf71",
+    "rcuk________": "00dq2kk65",  # deprecated funder org
     "sfi_________": "0271asj38",
     "snsf________": "00yjd3n13",
     "sshrc_______": "006cvnv84",
     "tubitakf____": "04w9kkr77",
+    "twcf________": "00x0z1472",
+    "ukri________": "001aqnf71",
     "wt__________": "029chgv08",
 }
 """Mapping of OpenAIRE and ROR funder codes."""
@@ -98,6 +114,21 @@ VOCABULARIES_NAMES_SCHEMES = {
 }
 """Names allowed identifier schemes."""
 
+VOCABULARIES_SUBJECTS_SCHEMES = {
+    "gnd": {"label": _("GND"), "validator": idutils.is_gnd, "datacite": "GND"},
+    "url": {"label": _("URL"), "validator": idutils.is_url},
+}
+"""Subjects allowed identifier schemes."""
+
+VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [
+    "names",
+    "affiliations",
+    "awards",
+    "funders",
+    "subjects",
+]
+"""List of custom vocabulary types."""
+
 VOCABULARIES_DATASTREAM_READERS = {
     "csv": CSVReader,
     "json": JsonReader,
@@ -107,6 +138,7 @@ VOCABULARIES_DATASTREAM_READERS = {
     "yaml": YamlReader,
     "zip": ZipReader,
     "xml": XMLReader,
+    "oai-pmh": OAIPMHReader,
 }
 """Data Streams readers."""
 
@@ -118,5 +150,40 @@ VOCABULARIES_DATASTREAM_TRANSFORMERS = {
 VOCABULARIES_DATASTREAM_WRITERS = {
     "service": ServiceWriter,
     "yaml": YamlWriter,
+    "async": AsyncWriter,
 }
 """Data Streams writers."""
+
+VOCABULARIES_TYPES_SORT_OPTIONS = {
+    "name": dict(
+        title=_("Name"),
+        fields=["id"],
+    ),
+    "count": dict(
+        title=_("Number of entries"),
+        fields=["count"],
+    ),
+}
+"""Definitions of available Vocabulary types sort options. """
+
+VOCABULARIES_TYPES_SEARCH = {
+    "facets": [],
+    "sort": ["name", "count"],
+}
+"""Vocabulary type search configuration."""
+
+SUBJECTS_EUROSCIVOC_FILE_URL = "https://op.europa.eu/o/opportal-service/euvoc-download-handler?cellarURI=http%3A%2F%2Fpublications.europa.eu%2Fresource%2Fdistribution%2Feuroscivoc%2F20231115-0%2Frdf%2Fskos_ap_eu%2FEuroSciVoc-skos-ap-eu.rdf&fileName=EuroSciVoc-skos-ap-eu.rdf"
+"""Subject EuroSciVoc file download link."""
+
+VOCABULARIES_ORCID_ACCESS_KEY = "TODO"
+"""ORCID access key to access the s3 bucket."""
+VOCABULARIES_ORCID_SECRET_KEY = "TODO"
+"""ORCID secret key to access the s3 bucket."""
+VOCABULARIES_ORCID_SUMMARIES_BUCKET = "v3.0-summaries"
+"""ORCID summaries bucket name."""
+VOCABULARIES_ORCID_SYNC_MAX_WORKERS = 32
+"""ORCID max number of simultaneous workers/connections."""
+VOCABULARIES_ORCID_SYNC_SINCE = {
+    "days": 1,
+}
+"""ORCID time shift to sync. Parameters accepted are the ones passed to 'datetime.timedelta'."""

invenio_vocabularies/contrib/affiliations/affiliations.py

@@ -8,12 +8,14 @@
 
 """Vocabulary affiliations."""
 
+from flask_resources import JSONSerializer, ResponseHandler
+from invenio_db import db
 from invenio_records.dumpers import SearchDumper
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records_resources.factories.factory import RecordTypeFactory
+from invenio_records_resources.records.systemfields import ModelPIDField
+from invenio_records_resources.resources.records.headers import etag_headers
 
-from ...records.pidprovider import PIDProviderFactory
-from ...records.systemfields import BaseVocabularyPIDFieldContext
 from ...services.permissions import PermissionPolicy
 from .config import AffiliationsSearchOptions, service_components
 from .schema import AffiliationSchema
@@ -21,17 +23,23 @@ from .schema import AffiliationSchema
 record_type = RecordTypeFactory(
     "Affiliation",
     # Data layer
+    pid_field_cls=ModelPIDField,
     pid_field_kwargs={
-        "create": False,
-        "provider": PIDProviderFactory.create(pid_type="aff"),
-        "context_cls": BaseVocabularyPIDFieldContext,
+        "model_field_name": "pid",
+    },
+    model_cls_attrs={
+        # cannot set to nullable=False because it would fail at
+        # service level when create({}), see records-resources.
+        "pid": db.Column(db.String(255), unique=True),
     },
     schema_version="1.0.0",
     schema_path="local://affiliations/affiliation-v1.0.0.json",
+    index_name="affiliations-affiliation-v2.0.0",
     record_dumper=SearchDumper(
+        model_fields={"pid": ("id", str)},
         extensions=[
             IndexedAtDumperExt(),
-        ]
+        ],
     ),
     # Service layer
     service_id="affiliations",
@@ -41,4 +49,12 @@ record_type = RecordTypeFactory(
     permission_policy_cls=PermissionPolicy,
     # Resource layer
     endpoint_route="/affiliations",
+    resource_cls_attrs={
+        "response_handlers": {
+            "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+            "application/vnd.inveniordm.v1+json": ResponseHandler(
+                JSONSerializer(), headers=etag_headers
+            ),
+        }
+    },
 )

invenio_vocabularies/contrib/affiliations/api.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -8,7 +8,6 @@
 
 """Vocabulary affiliations."""
 
-
 from .affiliations import record_type
 
 Affiliation = record_type.record_cls

invenio_vocabularies/contrib/affiliations/config.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2024 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -9,6 +9,7 @@
 """Vocabulary affiliations configuration."""
 
 from flask import current_app
+from invenio_i18n import get_locale
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services import SearchOptions
 from invenio_records_resources.services.records.components import DataComponent
@@ -20,6 +21,7 @@ from ...services.components import PIDComponent
 affiliation_schemes = LocalProxy(
     lambda: current_app.config["VOCABULARIES_AFFILIATION_SCHEMES"]
 )
+localized_title = LocalProxy(lambda: f"title.{get_locale()}^20")
 
 
 class AffiliationsSearchOptions(SearchOptions):
@@ -28,11 +30,14 @@ class AffiliationsSearchOptions(SearchOptions):
     suggest_parser_cls = SuggestQueryParser.factory(
         fields=[
             "name^100",
-            "acronym^20",
-            "title.*^5",
-            "title.*._2gram",
-            "title.*._3gram",
+            "acronym.keyword^100",
+            "acronym^40",
+            localized_title,
+            "id^20",
+            "aliases^20",
         ],
+        type="most_fields",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-multi-match-query.html#multi-match-types
+        fuzziness="AUTO",  # https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness
     )
 
     sort_default = "bestmatch"