hermes-plugin-rsd 0.1.0__py3-none-any.whl

hermes_plugin_rsd/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+# SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2

hermes_plugin_rsd/deposit.py

@@ -0,0 +1,516 @@
+# SPDX-FileCopyrightText: 2025 - 2026 GFZ Helmholtz Centre for Geosciences
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+import logging
+import os
+import re
+
+from hermes.commands.deposit.base import (
+    BaseDepositPlugin,
+    CodeMetaContext,
+    HermesDepositCommand,
+)
+from pydantic_core import ValidationError
+
+from hermes_plugin_rsd.models.contributor import Contributor
+from hermes_plugin_rsd.models.keyword import Keyword
+from hermes_plugin_rsd.models.keyword_for_software import KeywordForSoftware
+from hermes_plugin_rsd.models.license_for_software import LicenseForSoftware
+from hermes_plugin_rsd.models.organisation import Organisation
+from hermes_plugin_rsd.models.repository_url import RepositoryUrl
+from hermes_plugin_rsd.models.rsd_software_entry import RsdSoftwareEntry
+from hermes_plugin_rsd.models.software import Software
+from hermes_plugin_rsd.models.software_for_organisation import SoftwareForOrganisation
+from hermes_plugin_rsd.settings import RsdPluginSettings
+from hermes_plugin_rsd.utils.misc import get_new_objects
+from hermes_plugin_rsd.utils.ror import RorResolver, create_organisation_from_ror_entry
+from hermes_plugin_rsd.utils.rsd_api_handler import RsdApiHandler
+from hermes_plugin_rsd.utils.rsd_client import RsdClient
+
+log = logging.getLogger("hermes.deposit.rsd")
+
+SOFTWARE_SHORT_STATEMENT_MAX_LENGTH = 300
+
+
+def spdx_identifier_from_url(url: str) -> str | None:
+    prefix = "https://spdx.org/licenses/"
+    if url.startswith(prefix):
+        return url.removeprefix(prefix)
+    return url
+
+
+def truncate_short_statement(
+    sentence: str, max_len: int = SOFTWARE_SHORT_STATEMENT_MAX_LENGTH
+):
+    suffix = " …"
+    if len(sentence) <= max_len:
+        return sentence
+    if " " in sentence[max_len - 2 : max_len - 1]:
+        return sentence[: max_len - 1] + suffix
+    words = sentence.split(" ")
+    for n in range(len(words), 0, -1):
+        short_statement = " ".join(words[:n]).strip(":,") + suffix
+        if len(short_statement) <= max_len:
+            return short_statement
+    raise RuntimeError("Could not shorten sentence: ", sentence)
+
+
+def split_name(name: str) -> list[str]:
+    if ", " in name:
+        family_name, given_name = name.split(", ")
+    elif " " in name:
+        given_name, family_name = name.split(" ")
+    else:
+        log.error(
+            "Cannot split name '%s'. Please provide a mapping in hermes.toml.", name
+        )
+        raise ValueError("Cannot split name '%s'", name)
+    return [given_name, family_name]
+
+
+class RsdDepositPlugin(BaseDepositPlugin):
+    """RSD Deposit Plugin"""
+
+    keywords: list[Keyword] = []
+    organisations: list[Organisation] = []
+    settings_class = RsdPluginSettings
+    settings: RsdPluginSettings = None
+    software_entry: RsdSoftwareEntry = None
+    rsd_client = None
+    rsd_api_handler = None
+    ror_resolver = None
+
+    def __init__(self, command: HermesDepositCommand, ctx: CodeMetaContext) -> None:
+        super().__init__(command, ctx)
+        self.settings = self.command.settings.rsd
+        api_token = os.environ.get("RSD_API_TOKEN", "")
+
+        if api_token == "":
+            raise ValueError("Environment variable RSD_API_TOKEN is undefined.")
+
+        self.rsd_api_handler = RsdApiHandler(
+            RsdClient(api_token=api_token, rsd_url=self.settings.rsd_url)
+        )
+        self.ror_resolver = RorResolver()
+
+    def __call__(self, command: HermesDepositCommand):
+        """__call__
+        Overwrite base class __call__ as it does not fit to this plugin.
+
+        Minimum required data for the first-time usage:
+
+        - rsd url
+        - rsd token
+        - desired software slug
+
+        Worklfow:
+        validate metadata
+        if is initial version
+        - check availability of desired software name and slug
+        - set publish-method to POST
+        if not:
+        - set publish-method to PUT
+        map metadata
+        publish data
+        - if initial version: warning that software must be set to publicly visible in
+          RSD manually
+        """
+        self.prepare()
+
+        if self.is_initial_publication():
+            self.create_initial_version()
+
+        self.map_metadata()
+        self.publish()
+
+    def prepare(self) -> None:
+        self.validate()
+        self.software_entry = self.rsd_api_handler.get_representation_from_rsd(
+            self.settings.software_slug
+        )
+
+    def validate(self) -> None:
+        self.validate_slug()
+        self.validate_url()
+        self.validate_repository_type()
+
+    def validate_slug(self) -> None:
+        # Definition see
+        # https://github.com/research-software-directory/RSD-as-a-service/blob/main/database/004-create-software-table.sql
+
+        if len(self.settings.software_slug) > 250:
+            raise ValueError("software_slug is longer than 250 characters.")
+        pattern = r"^[a-z0-9]+(-[a-z0-9]+)*$"
+        if not re.match(pattern, self.settings.software_slug):
+            raise ValueError("Invalid software_slug.")
+
+    def validate_url(self) -> None:
+        if self.settings.rsd_url is None:
+            raise ValueError("The parameter rsd_url must be provided.")
+        if not self.settings.rsd_url.startswith(("http://", "https://")):
+            raise ValueError("URL must start with 'http://' or 'https://'")
+        self.settings.rsd_url = self.settings.rsd_url.removesuffix("/")
+
+    def validate_repository_type(self) -> None:
+        repository_type = self.ctx["codemeta"].get("@type", None)
+        if repository_type != "SoftwareSourceCode":
+            log.error(
+                "Repository must be source code to be published, but is type %s",
+                repository_type,
+            )
+            raise ValidationError("Invalid @type. Must be SoftwareSourceCode.")
+
+    def is_initial_publication(self) -> bool:
+        if self.software_entry.software.id is None:
+            log.info("Software entry does not exist yet.")
+            return True
+        log.info(
+            "Software entry with id %s already exists.",
+            self.software_entry.software.id,
+        )
+        return False
+
+    def create_initial_version(self) -> None:
+        log.info("Creating initial representation in the RSD.")
+        self.software_entry.software = Software(
+            brand_name="Initial version", slug=self.settings.software_slug
+        )
+        self.software_entry.software.id = (
+            self.rsd_api_handler.create_or_update_software(self.software_entry.software)
+        )
+        log.info("Software has id %s", self.software_entry.software.id)
+
+    def map_metadata(self) -> None:
+        """
+        Create a new RsdSoftwareEntry object that holds the metadata collected by
+        HERMES. This object is later merged with the existing or new RsdSoftwareEntry.
+        """
+
+        """
+        Process codemeta context
+        - [ ] author, cff
+        - [x] contributors, git
+        - [-] date published, cff
+        - [x] description, cff (cff prop: abstract) => short_statement
+        - [x] identifier, cff (doi, identifiers)
+        - [x] keywords, cff
+        - [x] license (as url referencing to spdx.org), cff
+        - [x] name, cff (cff prop: title)
+        - [x] type, cff (Dataset, SoftwareSourceCode)
+        - [x] urls, cff
+        - [-] version, cff
+        """
+        codemeta = self.ctx["codemeta"]
+
+        self.map_software_entry(codemeta)
+        self.map_contributors(codemeta)
+        self.map_repository_url(codemeta)
+        self.map_license(codemeta)
+        self.map_keywords(codemeta)
+        self.infer_organisations()
+
+    def infer_organisations(self):
+        """Infers ROR organisations based on email addresses of contributors.
+
+        If an organisation is found it will be added to self.organisations
+        """
+        for contributor in self.software_entry.contributors:
+            if not contributor.email_address:
+                continue
+            domain = contributor.email_address.split("@")[1]
+            ror_organisation = self.ror_resolver.search_organisation_by_domain(domain)
+            if ror_organisation:
+                # Check if already exists in RSD
+                organisation = self.rsd_api_handler.get_organisation(
+                    ror_id=ror_organisation["id"]
+                )
+                if organisation is None:
+                    organisation = create_organisation_from_ror_entry(ror_organisation)
+                if organisation not in self.organisations:
+                    # TODO: check if this should be moved to self.software_entry
+                    self.organisations.append(organisation)
+            else:
+                log.info(
+                    "Could not find an associated organisation for %s",
+                    contributor.email_address,
+                )
+
+        for organisation in self.organisations:
+            if organisation not in self.software_entry.software_for_organisation:
+                self.software_entry.software_for_organisation.append(
+                    SoftwareForOrganisation(
+                        organisation=organisation,
+                        software=self.software_entry.software.id,
+                    )
+                )
+
+    def map_keywords(self, codemeta):
+        if self.software_entry.software.id is None:
+            raise ValueError("Software ID must be defined to map keywords.")
+        keywords = codemeta.get("keywords", [])
+        for keyword in keywords:
+            # Check if keyword already in software entry
+            if keyword in self.software_entry.keywords:
+                continue
+            # Check if keyword already exists in the RSD
+            # If yes, use keyword ID
+            keyword_reference = self.rsd_api_handler.get_keyword(keyword)
+            if keyword_reference:
+                # Keyword exists, reference it
+                new_keyword_for_software = KeywordForSoftware(
+                    software=self.software_entry.software.id,
+                    keyword=keyword_reference,
+                )
+                if new_keyword_for_software not in self.software_entry.keywords:
+                    self.software_entry.keywords.append(new_keyword_for_software)
+            else:
+                # Keyword does not exist yet, we need to create it
+                # this also means it does not exist in the entry yet
+                new_keyword = Keyword(value=keyword, id=None)
+                self.keywords.append(new_keyword)
+                self.software_entry.keywords.append(
+                    KeywordForSoftware(
+                        software=self.software_entry.software.id,
+                        keyword=new_keyword,
+                    )
+                )
+
+    def map_license(self, codemeta):
+        licences = codemeta.get("license", None)
+        spdx_prefix = "https://spdx.org/licenses/"
+        if type(licences) is str:
+            licences = [licences]
+        for licence_data in licences:
+            open_source = False
+            licence = licence_data
+            if licence.startswith(spdx_prefix):
+                open_source = True
+                licence = licence_data.removeprefix(spdx_prefix)
+            new_licence = LicenseForSoftware(
+                license=licence,
+                open_source=open_source,
+                software=self.software_entry.software.id,
+            )
+            if new_licence not in self.software_entry.license:
+                self.software_entry.license.append(new_licence)
+
+    def map_repository_url(self, codemeta):
+        code_repository = codemeta.get("codeRepository", None)
+        if not code_repository:
+            return
+
+        code_platform = self.rsd_api_handler.get_code_platform(code_repository)
+        log.debug("Identified code platform: %s", code_platform)
+
+        if not self.software_entry.repository_url:
+            self.software_entry.repository_url = RepositoryUrl(
+                software=self.software_entry.software.id,
+                url=code_repository,
+                code_platform=code_platform,
+            )
+        else:
+            self.software_entry.repository_url.url = code_repository
+            self.software_entry.repository_url.code_platform = code_platform
+
+    def map_contributors(self, codemeta):
+        # Map Contributors
+        contributors = codemeta.get("contributor", [])
+
+        # Process Contributors
+        # https://schema.org/contributor
+        # Expected types: People, Organizations
+        if len(contributors) > 0:
+            for contributor in contributors:
+                contributor_type = contributor.get("@type", None)
+                if contributor_type == "Person":
+                    self.map_person_metadata(contributor)
+                elif contributor_type == "Organization":
+                    # TODO: Process organisation
+                    pass
+                else:
+                    log.error("Invalid contributor type: %s", contributor_type)
+                    raise TypeError(f"Invalid contributor type: {contributor_type}")
+
+    def map_software_entry(self, codemeta):
+        """Map codemeta data to Software
+
+        Metadata in the ex
+
+        - [x] brand_name
+        - [x] concept_doi
+        - [x] description
+        - [x] short_statement
+        - [x] get_started_url
+        """
+        description = codemeta.get("description", None)
+        software = self.software_entry.software
+
+        if description:
+            if len(description) > SOFTWARE_SHORT_STATEMENT_MAX_LENGTH:
+                log.warning(
+                    "Description length is > %s characters. Shortening description to "
+                    "fit into short description.",
+                    SOFTWARE_SHORT_STATEMENT_MAX_LENGTH,
+                )
+                software.short_statement = truncate_short_statement(description)
+            software.description = description
+
+        title = codemeta.get("name", None)
+        if title:
+            software.brand_name = title
+        else:
+            log.error("A name is required to create an RSD entry.")
+            raise ValidationError("Software title undefined.")
+
+        software.get_started_url = codemeta.get("url", None)
+
+        identifier = codemeta.get("identifier", None)
+        if identifier:
+            if identifier.startswith("https://doi.org/"):
+                log.warning(
+                    "Found an identifier in codemeta context. "
+                    "Assuming this is the concept DOI of the work."
+                )
+                identifier = identifier.removeprefix("https://doi.org/")
+            if re.search(r"^10.\d{4,}/\S+", identifier):
+                log.debug("%s seems to be a DOI.", identifier)
+                software.concept_doi = identifier
+            elif identifier is not None:
+                log.warning(
+                    "Identifier %s is not a DOI and is unsupported.", identifier
+                )
+
+    def map_person_metadata(self, person: dict) -> None:
+        """
+        Maps person metadata and creates contributors for each person.
+
+        The `hermes.harvest.git` plugin does not return family and given name, only the
+        complete author name. We use the following strategy for splitting:
+
+        - One space inside "name" => split into `[givenName] [familyName]`
+        - ", " inside "name" => split into `[familyName], [givenName]`
+        - else: require manual changes
+
+        Attribute mapping
+        -----------------
+
+        | RSD Attribute   | CodeMeta   |
+        |-----------------|------------|
+        | family_names    | familyName |
+        | given_names     | givenName  |
+
+        Parameters
+        ----------
+        person
+            Dictionary containing person metadata
+
+        Merge strategy
+        --------------
+        Adds a new contributor to the list if:
+
+        - no contributor with the same `given_names` and `family_names` exists
+        - or if no contributor with the same ORCiD exists
+
+        This means changes to other attributes are not regarded, especially changes in
+
+        - affiliation
+        - email address
+        - contact person
+        - role
+
+        Raises
+        ------
+        RuntimeError
+            If name of person cannot be detected.
+        ValidationError
+            If person attributes are missing or invalid.
+        """
+        family_name = person.get("familyName")
+        given_name = person.get("givenName")
+        if family_name is None or given_name is None:
+            log.info("familyName or givenName undefined for person: %s", person)
+            name = person.get("name")
+            if name is None:
+                log.error("Unable to detect name for person: %s", person)
+                raise RuntimeError("Unable to detect name for person.")
+            if name == "GitHub":
+                log.info(
+                    "hermes.harvest.git detected contributions by GitHub. "
+                    "Ignoring as these will not be listed."
+                )
+                return
+            given_name, family_name = split_name(person["name"])
+
+        try:
+            log.info(
+                "Generating contributor for givenName: %s and familyName: %s.",
+                given_name,
+                family_name,
+            )
+            contributor = Contributor(
+                family_names=family_name,
+                given_names=given_name,
+                software=self.software_entry.software.id,
+            )
+            contributor.email_address = person.get("email")
+            if contributor not in self.software_entry.contributors:
+                self.software_entry.contributors.append(contributor)
+            else:
+                log.info(
+                    "Contributor %s already exists in the RSD entry, not updating.",
+                    contributor,
+                )
+        except ValidationError as e:
+            log.error(
+                "%d validation error(s) while processing Contributor %s:",
+                len(e.errors()),
+                person,
+            )
+            for error in e.errors():
+                log.error(
+                    "Required attribute is missing or invalid: %s", error["loc"][0]
+                )
+            raise e
+
+    def publish(self):
+        # TODO: do not stop execution when adding relations to software. collect the
+        #  errors and generate a meaningful error message at the end of the publish
+        #  process
+        # At this point the software representation definitely exists in the RSD
+        self.rsd_api_handler.create_or_update_software(self.software_entry.software)
+
+        if self.software_entry.repository_url is not None:
+            self.rsd_api_handler.create_or_update_repository_url(
+                self.software_entry.repository_url
+            )
+
+        new_licenses = get_new_objects(self.software_entry.license)
+        for licence in new_licenses:
+            self.rsd_api_handler.create_license_for_software(licence)
+
+        new_keywords = get_new_objects(self.keywords)
+        for keyword in new_keywords:
+            self.rsd_api_handler.create_keyword(keyword)
+
+        new_keywords_for_software = get_new_objects(self.software_entry.keywords)
+        for keyword_for_software in new_keywords_for_software:
+            self.rsd_api_handler.create_keyword_for_software(keyword_for_software)
+
+        new_contributors = get_new_objects(self.software_entry.contributors)
+        for contributor in new_contributors:
+            self.rsd_api_handler.create_contributor(contributor)
+
+        new_sw_for_org = get_new_objects(self.software_entry.software_for_organisation)
+        for software_for_organisation in new_sw_for_org:
+            self.rsd_api_handler.create_software_for_organisation(
+                software_for_organisation
+            )
+
+        log.info(
+            "\n*************************************\n"
+            "Software entry successfully created. Please review the software entry "
+            "and publish it so that it will be discoverable.\n"
+            "*************************************\n"
+        )

hermes_plugin_rsd/harvest.py

@@ -0,0 +1,69 @@
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+# SPDX-FileCopyrightText: 2026 GFZ Helmholtz Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+import logging
+import os
+
+from hermes.commands import HermesHarvestCommand
+from hermes.commands.harvest.base import HermesHarvestPlugin
+from hermes.model.errors import HermesValidationError
+
+from hermes_plugin_rsd.settings import RsdPluginSettings
+from hermes_plugin_rsd.utils.rsd_api_handler import RsdApiHandler
+from hermes_plugin_rsd.utils.rsd_client import RsdClient
+
+log = logging.getLogger("cli.harvest.rsd")
+
+
+class RsdHarvestPlugin(HermesHarvestPlugin):
+    settings_class = RsdPluginSettings
+    command: HermesHarvestCommand
+    rsd_api_handler: RsdApiHandler
+    settings = None
+
+    def __call__(self, command: HermesHarvestCommand):
+        self.command = command
+        self.settings = self.command.settings.rsd
+        api_token = os.environ.get("RSD_API_TOKEN", "")
+
+        if api_token == "":
+            log.warning(
+                "RSD_API_TOKEN is undefined, unable to access unpublished software "
+                "entries."
+            )
+
+        self.rsd_api_handler = RsdApiHandler(
+            RsdClient(api_token=api_token, rsd_url=self.settings.rsd_url)
+        )
+
+        """
+        Harvests relevant metadata from an RSD entry.
+
+        Latest supported and tested RSD version: v5.4.4
+
+        The following endpoints are fetched:
+
+        - [x] software
+        - [x] repository_url
+        - [ ] category_for_software
+        - [x] keyword_for_software
+        - [x] license_for_software
+        - [ ] mention_for_software
+        - [ ] reference_paper_for_software
+        - [ ] software_for_software
+        - [x] swhid_for_software
+
+        The output will be digested by HermesHarvestContext.update_from().
+        """
+
+        slug = self.settings.software_slug
+        rsd_entry = self.rsd_api_handler.get_representation_from_rsd(slug)
+        if not rsd_entry.exists():
+            raise HermesValidationError(
+                "Software entry does not exist in", self.settings.rsd_url
+            )
+        harvested_data = rsd_entry.to_codemeta()
+        tags = {"rsd_url": self.settings.rsd_url + "/software/" + slug}
+        return harvested_data, tags

hermes_plugin_rsd/models/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+# SPDX-FileCopyrightText: 2025 GFZ Helmholtz Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2

hermes_plugin_rsd/models/contributor.py

@@ -0,0 +1,81 @@
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+# SPDX-FileCopyrightText: 2025, 2026 GFZ Helmholtz Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+import re
+import uuid
+from typing import Annotated
+
+from pydantic import PlainSerializer, ValidationError, field_validator
+
+from hermes_plugin_rsd.models.rsd_base_model import RsdBaseModel
+from hermes_plugin_rsd.utils.misc import serialise_uuid
+
+
+class Contributor(RsdBaseModel):
+    # See
+    # https://github.com/research-software-directory/RSD-as-a-service/blob/main/database/005-create-relations-for-software.sql
+    # Required attributes
+    family_names: str
+    given_names: str
+    software: Annotated[uuid.UUID, PlainSerializer(serialise_uuid)]
+    # Optional
+    affiliation: str | None = None
+    email_address: str | None = None
+    is_contact_person: bool = False
+    orcid: str | None = None
+    role: str | None = None
+
+    def __str__(self):
+        return f"{self.given_names} {self.family_names}"
+
+    def __eq__(self, other):
+        if isinstance(other, Contributor):
+            if self.orcid and other.orcid:
+                return self.orcid == other.orcid
+            if (
+                self.family_names
+                and self.given_names
+                and other.family_names
+                and other.given_names
+            ):
+                return (
+                    self.family_names == other.family_names
+                    and self.given_names == other.given_names
+                )
+        return False
+
+    def __hash__(self):
+        return hash((self.family_names, self.given_names, self.orcid))
+
+    @field_validator("email_address")
+    @classmethod
+    def validate_email(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        if not re.match(r"[^@]+@[^@]+\.[^@]+", value):
+            raise ValidationError("Invalid e-mail address: %s", value)
+        return value
+
+    @field_validator("orcid")
+    @classmethod
+    def validate_orcid(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        if not re.match(r"^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]{1}$", value):
+            raise ValidationError("Invalid ORCID: %s", value)
+        return value
+
+    def to_codemeta(self):
+        person = {
+            "@type": "Person",
+            "name": self.given_names + " " + self.family_names,
+        }
+        if self.affiliation:
+            person["affiliation"] = {"@type": "Organization", "name": self.affiliation}
+        if self.email_address:
+            person["email"] = self.email_address
+        if self.orcid:
+            person["identifier"] = self.orcid
+        return person

hermes_plugin_rsd/models/keyword.py

@@ -0,0 +1,20 @@
+# SPDX-FileContributor: Christian Meeßen (GFZ) <christian.meessen@gfz.de>
+# SPDX-FileCopyrightText: 2026 GFZ Helmholtz Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+import uuid
+from typing import Annotated
+
+from pydantic import PlainSerializer
+
+from hermes_plugin_rsd.models.rsd_base_model import RsdBaseModel
+from hermes_plugin_rsd.utils.misc import serialise_uuid
+
+
+class Keyword(RsdBaseModel):
+    id: Annotated[uuid.UUID | None, PlainSerializer(serialise_uuid)] = None
+    value: str
+
+    def __str__(self):
+        return self.value