hermes 0.1.0__py3-none-any.whl

hermes/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0

hermes/__main__.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+from hermes.commands.cli import main
+
+
+if __name__ == "__main__":
+    main()

hermes/commands/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+# This is an interface file that only provides a public interface, hence linter is disabled to avoid
+# "unused import" errors.
+# flake8: noqa
+
+from hermes.commands.base import HermesHelpCommand
+from hermes.commands.clean.base import HermesCleanCommand
+from hermes.commands.curate.base import HermesCurateCommand
+from hermes.commands.harvest.base import HermesHarvestCommand
+from hermes.commands.process.base import HermesProcessCommand
+from hermes.commands.deposit.base import HermesDepositCommand
+from hermes.commands.postprocess.base import HermesPostprocessCommand

hermes/commands/base.py

@@ -0,0 +1,199 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+import abc
+import argparse
+import logging
+import pathlib
+from importlib import metadata
+from typing import Dict, Optional, Type
+
+import toml
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class HermesSettings(BaseSettings):
+    """Root class for HERMES configuration model."""
+
+    model_config = SettingsConfigDict(env_file_encoding='utf-8')
+
+    logging: Dict = {}
+
+
+class HermesCommand(abc.ABC):
+    """Base class for a HERMES workflow command.
+
+    :cvar NAME: The name of the sub-command that is defined here.
+    """
+
+    command_name: str = ""
+    settings_class: Type = HermesSettings
+
+    def __init__(self, parser: argparse.ArgumentParser):
+        """Initialize a new instance of any HERMES command.
+
+        :param parser: The command line parser used for reading command line arguments.
+        """
+        self.parser = parser
+        self.plugins = self.init_plugins()
+        self.settings = None
+
+        self.log = logging.getLogger(f"hermes.{self.command_name}")
+
+    def init_plugins(self):
+        """Collect and initialize the plugins available for the HERMES command."""
+
+        # Collect all entry points for this group (i.e., all valid plug-ins for the step)
+        entry_point_group = f"hermes.{self.command_name}"
+        group_plugins = {
+            entry_point.name: entry_point.load()
+            for entry_point in metadata.entry_points(group=entry_point_group)
+        }
+
+        # Collect the plug-in specific configurations
+        self.derive_settings_class({
+            plugin_name: plugin_class.settings_class
+            for plugin_name, plugin_class in group_plugins.items()
+            if hasattr(plugin_class, "settings_class") and plugin_class.settings_class is not None
+        })
+
+        return group_plugins
+
+    @classmethod
+    def derive_settings_class(cls, setting_types: Dict[str, Type]) -> None:
+        """Build a new Pydantic data model class for configuration.
+
+        This will create a new class that includes all settings from the plugins available.
+        """
+
+        if cls.settings_class is not None:
+            # Derive a new settings model class that contains all the plug-in extensions
+            cls.settings_class = type(
+                f"{cls.__name__}Settings",
+                (cls.settings_class, ),
+                {
+                    **{
+                        plugin_name: plugin_settings()
+                        for plugin_name, plugin_settings in setting_types.items()
+                        if plugin_settings is not None
+                    },
+                    "__annotations__": setting_types,
+                },
+            )
+        elif setting_types:
+            raise ValueError(f"Command {cls.command_name} has no settings, hence plugin must not have settings, too.")
+
+    def init_common_parser(self, parser: argparse.ArgumentParser) -> None:
+        """Initialize the common command line arguments available for all HERMES sub-commands.
+
+        :param parser: The base command line parser used as entry point when reading command line arguments.
+        """
+
+        parser.add_argument(
+            "--path", default=pathlib.Path(), type=pathlib.Path, help="Working path"
+        )
+        parser.add_argument(
+            "--config",
+            default=pathlib.Path("hermes.toml"),
+            type=pathlib.Path,
+            help="Configuration file in TOML format",
+        )
+
+        plugin_args = parser.add_argument_group("Extra options")
+        plugin_args.add_argument(
+            "-O",
+            nargs=2,
+            action="append",
+            default=[],
+            metavar=("NAME", "VALUE"),
+            dest="options",
+            help="Configuration values to override hermes.toml options. "
+            "NAME is the dotted name / path to the option in the TOML file, "
+            "VALUE is the actual value.",
+        )
+
+    def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
+        """Initialize the command line arguments available for this specific HERMES sub-commands.
+
+        You should override this method to add your custom arguments to the command line parser of
+        the respective sub-command.
+
+        :param command_parser: The command line sub-parser responsible for the HERMES sub-command.
+        """
+
+        pass
+
+    def load_settings(self, args: argparse.Namespace):
+        """Load settings from the configuration file (passed in from command line)."""
+
+        toml_data = toml.load(args.path / args.config)
+        self.root_settings = HermesCommand.settings_class.model_validate(toml_data)
+        self.settings = getattr(self.root_settings, self.command_name)
+
+    def patch_settings(self, args: argparse.Namespace):
+        """Process command line options for the settings."""
+
+        for key, value in args.options:
+            target = self.settings
+            sub_keys = key.split('.')
+            for sub_key in sub_keys[:-1]:
+                target = getattr(target, sub_key)
+
+            # TODO: Transform the value accordingly before setting it
+            setattr(target, sub_keys[-1], value)
+
+    @abc.abstractmethod
+    def __call__(self, args: argparse.Namespace):
+        """Execute the HERMES sub-command.
+
+        :param args: The namespace that was returned by the command line parser when reading the arguments.
+        """
+
+        pass
+
+
+class HermesPlugin(abc.ABC):
+    """Base class for all HERMES plugins."""
+
+    settings_class: Optional[Type] = None
+
+    @abc.abstractmethod
+    def __call__(self, command: HermesCommand) -> None:
+        """Execute the plugin.
+
+        :param command: The command that triggered this plugin to run.
+        """
+
+        pass
+
+
+class HermesHelpSettings(BaseModel):
+    pass
+
+
+class HermesHelpCommand(HermesCommand):
+    """Show help page and exit."""
+
+    command_name = "help"
+    settings_class = HermesHelpSettings
+
+    def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
+        command_parser.add_argument(
+            "subcommand",
+            nargs="?",
+            metavar="COMMAND",
+            help="The HERMES sub-command to get help for.",
+        )
+
+    def __call__(self, args: argparse.Namespace) -> None:
+        if args.subcommand:
+            # When a sub-command is given, show its help page (i.e., by "running" the command with "-h" flag).
+            self.parser.parse_args([args.subcommand, "-h"])
+        else:
+            # Otherwise, simply show the general help and exit (cleanly).
+            self.parser.print_help()
+            self.parser.exit()

hermes/commands/clean/base.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+import argparse
+import shutil
+
+from pydantic import BaseModel
+
+from hermes.commands.base import HermesCommand
+
+
+class HermesCleanSettings(BaseModel):
+    """Configuration of the ``clean`` command."""
+    pass
+
+
+class HermesCleanCommand(HermesCommand):
+    """ Clean up caches from previous HERMES runs. """
+
+    command_name = "clean"
+    settings_class = HermesCleanSettings
+
+    def __call__(self, args: argparse.Namespace) -> None:
+        self.log.info("Removing HERMES caches...")
+
+        # Naive implementation for now... check errors, validate directory, don't construct the path ourselves, etc.
+        shutil.rmtree(args.path / '.hermes')

hermes/commands/cli.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Stephan Druskat
+# SPDX-FileContributor: Michael Meinel
+
+"""
+This module provides the main entry point for the HERMES command line application.
+"""
+import argparse
+
+from hermes.commands import HermesHelpCommand, HermesCleanCommand, HermesHarvestCommand, HermesProcessCommand, \
+                            HermesCurateCommand, HermesDepositCommand, HermesPostprocessCommand
+from hermes.commands.base import HermesCommand
+
+
+def main() -> None:
+    """
+    HERMES main entry point (i.e., run the CLI).
+
+    This command runs the selected HERMES sub-command.
+    """
+    parser = argparse.ArgumentParser(
+        prog="hermes",
+        description="This command runs HERMES workflow steps.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # Register all sub-commands to a new sub-parser each.
+    subparsers = parser.add_subparsers(dest="subcommand", required=True,
+                                       help="Available subcommands")
+    setting_types = {}
+
+    for command in (
+            HermesHelpCommand(parser),
+            HermesCleanCommand(parser),
+            HermesHarvestCommand(parser),
+            HermesProcessCommand(parser),
+            HermesCurateCommand(parser),
+            HermesDepositCommand(parser),
+            HermesPostprocessCommand(parser),
+    ):
+        if command.settings_class is not None:
+            setting_types[command.command_name] = command.settings_class
+
+        command_parser = subparsers.add_parser(command.command_name, help=command.__doc__)
+        command_parser.set_defaults(command=command)
+
+        command.init_common_parser(command_parser)
+        command.init_command_parser(command_parser)
+
+    # Construct the Pydantic Settings root model
+    HermesCommand.derive_settings_class(setting_types)
+
+    # Actually parse the command line, configure it and execute the selected sub-command.
+    args = parser.parse_args()
+
+    args.command.load_settings(args)
+    args.command.patch_settings(args)
+    args.command(args)

hermes/commands/curate/base.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+import argparse
+import os
+import shutil
+import sys
+
+from pydantic import BaseModel
+
+from hermes.commands.base import HermesCommand
+from hermes.model.context import CodeMetaContext
+
+
+class CurateSettings(BaseModel):
+    """Generic deposition settings."""
+
+    pass
+
+
+class HermesCurateCommand(HermesCommand):
+    """ Curate the unified metadata before deposition. """
+
+    command_name = "curate"
+    settings_class = CurateSettings
+
+    def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
+        pass
+
+    def __call__(self, args: argparse.Namespace) -> None:
+
+        self.log.info("# Metadata curation")
+
+        ctx = CodeMetaContext()
+        process_output = ctx.hermes_dir / 'process' / (ctx.hermes_name + ".json")
+
+        if not process_output.is_file():
+            self.log.error(
+                "No processed metadata found. Please run `hermes process` before curation."
+            )
+            sys.exit(1)
+
+        os.makedirs(ctx.hermes_dir / 'curate', exist_ok=True)
+        shutil.copy(process_output, ctx.hermes_dir / 'curate' / (ctx.hermes_name + '.json'))

hermes/commands/deposit/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2023 Helmholtz-Zentrum Dresden-Rossendorf (HZDR)
+#
+# SPDX-License-Identifier: Apache-2.0

hermes/commands/deposit/base.py

@@ -0,0 +1,145 @@
+# SPDX-FileCopyrightText: 2023 Helmholtz-Zentrum Dresden-Rossendorf (HZDR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: David Pape
+# SPDX-FileContributor: Michael Meinel
+
+import abc
+import argparse
+import json
+import sys
+
+from pydantic import BaseModel
+
+from hermes.commands.base import HermesCommand, HermesPlugin
+from hermes.model.context import CodeMetaContext
+from hermes.model.path import ContextPath
+from hermes.model.errors import HermesValidationError
+
+
+class BaseDepositPlugin(HermesPlugin):
+    """Base class that implements the generic deposition workflow.
+
+    TODO: describe workflow... needs refactoring to be less stateful!
+    """
+
+    def __init__(self, command, ctx):
+        self.command = command
+        self.ctx = ctx
+
+    def __call__(self, command: HermesCommand) -> None:
+        """Initiate the deposition process.
+
+        This calls a list of additional methods on the class, none of which need to be implemented.
+        """
+        self.command = command
+
+        self.prepare()
+        self.map_metadata()
+
+        if self.is_initial_publication():
+            self.create_initial_version()
+        else:
+            self.create_new_version()
+
+        self.update_metadata()
+        self.delete_artifacts()
+        self.upload_artifacts()
+        self.publish()
+
+    def prepare(self) -> None:
+        """Prepare the deposition.
+
+        This method may be implemented to check whether config and context match some initial conditions.
+
+        If no exceptions are raised, execution continues.
+        """
+        pass
+
+    @abc.abstractmethod
+    def map_metadata(self) -> None:
+        """Map the given metadata to the target schema of the deposition platform."""
+        pass
+
+    def is_initial_publication(self) -> bool:
+        """Decide whether to do an initial publication or publish a new version.
+
+        Returning ``True`` indicates that publication of an initial version will be executed, resulting in a call of
+        :meth:`create_initial_version`. ``False`` indicates a new version of an existing publication, leading to a call
+        of :meth:`create_new_version`.
+
+        By default, this returns ``True``.
+        """
+        return True
+
+    def create_initial_version(self) -> None:
+        """Create an initial version of the publication on the target platform."""
+        pass
+
+    def create_new_version(self) -> None:
+        """Create a new version of an existing publication on the target platform."""
+        pass
+
+    def update_metadata(self) -> None:
+        """Update the metadata of the newly created version."""
+        pass
+
+    def delete_artifacts(self) -> None:
+        """Delete any superfluous artifacts taken from the previous version of the publication."""
+        pass
+
+    def upload_artifacts(self) -> None:
+        """Upload new artifacts to the target platform."""
+        pass
+
+    @abc.abstractmethod
+    def publish(self) -> None:
+        """Publish the newly created deposit on the target platform."""
+        pass
+
+
+class DepositSettings(BaseModel):
+    """Generic deposition settings."""
+
+    target: str = ""
+
+
+class HermesDepositCommand(HermesCommand):
+    """ Deposit the curated metadata to repositories. """
+
+    command_name = "deposit"
+    settings_class = DepositSettings
+
+    def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
+        command_parser.add_argument('--file', '-f', nargs=1, action='append',
+                                    help="File that should be part of the deposition.")
+        command_parser.add_argument('--initial', action='store_true', default=False,
+                                    help="Allow initial deposition (i.e., minting a new PID).")
+
+    def __call__(self, args: argparse.Namespace) -> None:
+        self.args = args
+        plugin_name = self.settings.target
+        print(self.args)
+
+        ctx = CodeMetaContext()
+        codemeta_file = ctx.get_cache("curate", ctx.hermes_name)
+        if not codemeta_file.exists():
+            self.log.error("You must run the 'curate' command before deposit")
+            sys.exit(1)
+
+        codemeta_path = ContextPath("codemeta")
+        with open(codemeta_file) as codemeta_fh:
+            ctx.update(codemeta_path, json.load(codemeta_fh))
+
+        try:
+            plugin_func = self.plugins[plugin_name](self, ctx)
+
+        except KeyError:
+            self.log.error("Plugin '%s' not found.", plugin_name)
+
+        try:
+            plugin_func(self)
+
+        except HermesValidationError as e:
+            self.log.error("Error while executing %s: %s", plugin_name, e)

hermes/commands/deposit/error.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2023 Helmholtz-Zentrum Dresden-Rossendorf (HZDR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: David Pape
+
+class DepositionUnauthorizedError(Exception):
+    pass

hermes/commands/deposit/file.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2023 German Aerospace Center (DLR), Helmholtz-Zentrum Dresden-Rossendorf (HZDR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: David Pape
+# SPDX-FileContributor: Michael Meinel
+# SPDX-FileContributor: Stephan Druskat
+
+import json
+
+from pydantic import BaseModel
+
+from hermes.commands.deposit.base import BaseDepositPlugin
+from hermes.model.path import ContextPath
+
+
+class FileDepositSettings(BaseModel):
+    filename: str = 'hermes.json'
+
+
+class FileDepositPlugin(BaseDepositPlugin):
+    settings_class = FileDepositSettings
+
+    def map_metadata(self) -> None:
+        self.ctx.update(ContextPath.parse('deposit.file'), self.ctx['codemeta'])
+
+    def publish(self) -> None:
+        file_config = self.command.settings.file
+        output_data = self.ctx['deposit.file']
+
+        with open(file_config.filename, 'w') as deposition_file:
+            json.dump(output_data, deposition_file, indent=2)