snakemake-interface-software-deployment-plugins 0.1.0__tar.gz

snakemake_interface_software_deployment_plugins-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Snakemake Community
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

snakemake_interface_software_deployment_plugins-0.1.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.3
+Name: snakemake-interface-software-deployment-plugins
+Version: 0.1.0
+Summary: This package provides a stable interface for interactions between Snakemake and its software deployment plugins.
+License: MIT
+Author: Johannes Köster
+Author-email: johannes.koester@uni-due.de
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: argparse-dataclass (>=2.0.0,<3.0.0)
+Requires-Dist: snakemake-interface-common (>=1.17.4,<2.0.0)
+Description-Content-Type: text/markdown
+
+# Stable interfaces and functionality for Snakemake software deployment plugins
+
+This package provides a stable interface for interactions between Snakemake and its software deployment plugins.
+

snakemake_interface_software_deployment_plugins-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Stable interfaces and functionality for Snakemake software deployment plugins
+
+This package provides a stable interface for interactions between Snakemake and its software deployment plugins.

snakemake_interface_software_deployment_plugins-0.1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+authors = ["Johannes Köster <johannes.koester@uni-due.de>"]
+description = "This package provides a stable interface for interactions between Snakemake and its software deployment plugins."
+license = "MIT"
+name = "snakemake-interface-software-deployment-plugins"
+packages = [{include = "snakemake_interface_software_deployment_plugins"}]
+readme = "README.md"
+version = "0.1.0"
+
+[tool.poetry.dependencies]
+argparse-dataclass = "^2.0.0"
+python = "^3.11"
+snakemake-interface-common = "^1.17.4"
+
+[tool.poetry.group.dev.dependencies]
+coverage = {extras = ["toml"], version = "^6.3.1"}
+flake8-bugbear = "^22.1.11"
+pytest = "^7.0"
+ruff = "^0.9.9"
+
+[tool.coverage.run]
+omit = [".*", "*/site-packages/*"]
+
+[tool.coverage.report]
+fail_under = 63
+
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/__init__.py

@@ -0,0 +1,177 @@
+__author__ = "Johannes Köster"
+__copyright__ = "Copyright 2024, Johannes Köster"
+__email__ = "johannes.koester@uni-due.de"
+__license__ = "MIT"
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field, fields
+import hashlib
+from pathlib import Path
+import sys
+from typing import Optional
+import subprocess as sp
+
+from snakemake_interface_software_deployment_plugins.settings import (
+    SoftwareDeploymentSettingsBase,
+)
+
+
+_MANAGED_FIELDS = {
+    "settings",
+    "_managed_hash_store",
+    "_managed_deployment_hash_store",
+    "_obj_hash",
+}
+
+
+@dataclass
+class EnvSpecBase(ABC):
+    within: Optional["EnvSpecBase"]
+
+    @classmethod
+    def env_cls(cls):
+        return sys.modules[__name__].EnvBase
+
+
+@dataclass
+class EnvBase:
+    spec: EnvSpecBase
+    within: Optional["EnvBase"]
+    settings: Optional[SoftwareDeploymentSettingsBase]
+    _managed_hash_store: Optional[str] = field(init=False, default=None)
+    _managed_deployment_hash_store: Optional[str] = field(init=False, default=None)
+    _obj_hash: Optional[int] = field(init=False, default=None)
+
+    def __post_init__(self) -> None:  # noqa B027
+        """Do stuff after object initialization."""
+        pass
+
+    @abstractmethod
+    def decorate_shellcmd(self, cmd: str) -> str:
+        """Decorate given shell command such that it runs within the environment,
+        using self.spec.
+        """
+        ...
+
+    @abstractmethod
+    def record_hash(self, hash_object) -> None:
+        """Update given hash object such that it changes whenever the environment
+        specified via self.spec could potentially contain a different set of
+        software (in terms of versions or packages).
+        """
+        ...
+
+    def run_cmd(self, cmd: str, **kwargs) -> sp.CompletedProcess:
+        """Run a command while potentially respecting the self.within environment,
+        returning the result of subprocess.run.
+
+        kwargs is passed to subprocess.run, shell=True is always set.
+        """
+        if self.within is not None:
+            cmd = self.within.managed_decorate_shellcmd(cmd)
+        return sp.run(cmd, shell=True, **kwargs)
+
+    def managed_decorate_shellcmd(self, cmd: str) -> str:
+        cmd = self.decorate_shellcmd(cmd)
+        if self.within is not None:
+            cmd = self.within.managed_decorate_shellcmd(cmd)
+        return cmd
+
+    def hash(self) -> str:
+        return self._managed_generic_hash("hash")
+
+    def _managed_generic_hash(self, kind: str) -> str:
+        store = getattr(self, f"_managed_{kind}_store")
+        if store is None:
+            record_hash = f"record_{kind}"
+            hash_object = hashlib.md5()
+            if self.within is not None:
+                getattr(self.within, record_hash)(hash_object)
+            getattr(self, record_hash)(hash_object)
+            store = hash_object.hexdigest()
+        return store
+
+    def __hash__(self) -> int:
+        # take the hash of all fields by settings, _managed_hash_store and _managed_deployment_hash_store
+        if self._obj_hash is None:
+            self._obj_hash = hash(
+                tuple(
+                    getattr(self, field.name)
+                    for field in fields(self)
+                    if field.name not in _MANAGED_FIELDS
+                )
+            )
+        return self._obj_hash
+
+    def __eq__(self, other) -> bool:
+        return self.__class__ == other.__class__ and all(
+            getattr(self, field.name) == getattr(other, field.name)
+            for field in fields(self)
+            if field.name not in _MANAGED_FIELDS
+        )
+
+
+@dataclass
+class DeployableEnvBase(EnvBase):
+    _deployment_prefix: Optional[Path] = None
+
+    @abstractmethod
+    def deploy(self) -> None:
+        """Deploy the environment to self.deployment_path.
+
+        When issuing shell commands, the environment should use
+        self.provider.run(cmd: str) in order to ensure that it runs within eventual
+        parent environments (e.g. a container or an env module).
+        """
+        ...
+
+    @abstractmethod
+    def record_deployment_hash(self, hash_object) -> None:
+        """Update given hash such that it changes whenever the environment
+        needs to be redeployed, e.g. because its content has changed or the
+        deployment location has changed. The latter is only relevant if the
+        deployment is senstivive to the path (e.g. in case of conda, which patches
+        the RPATH in binaries).
+        """
+        ...
+
+    @abstractmethod
+    def remove(self) -> None:
+        """Remove the deployed environment."""
+        ...
+
+    def managed_deploy(self) -> None:
+        if isinstance(self, ArchiveableEnvBase) and self.archive_path.exists():
+            self.deploy_from_archive()
+        else:
+            self.deploy()
+
+    def deployment_hash(self) -> str:
+        return self._managed_generic_hash("deployment_hash")
+
+    @property
+    def deployment_path(self) -> Path:
+        return self.provider.deployment_prefix / self.deployment_hash()
+
+
+class ArchiveableEnvBase(EnvBase):
+    archive_prefix: Optional[Path] = None
+
+    @abstractmethod
+    def archive(self) -> None:
+        """Archive the environment to self.archive_path.
+
+        When issuing shell commands, the environment should use
+        self.provider.run(cmd: str) in order to ensure that it runs within eventual
+        parent environments (e.g. a container or an env module).
+        """
+        ...
+
+    @abstractmethod
+    def deploy_from_archive(self) -> None:
+        """Deploy the environment from an archive."""
+        ...
+
+    @property
+    def archive_path(self) -> Path:
+        return self.archive_prefix / self.hash()

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/_common.py

@@ -0,0 +1,10 @@
+__author__ = "Johannes Köster"
+__copyright__ = "Copyright 2024, Johannes Köster"
+__email__ = "johannes.koester@uni-due.de"
+__license__ = "MIT"
+
+
+software_deployment_plugin_prefix = "snakemake-software-deployment-plugin-"
+software_deployment_plugin_module_prefix = software_deployment_plugin_prefix.replace(
+    "-", "_"
+)

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/registry/__init__.py

@@ -0,0 +1,51 @@
+__author__ = "Johannes Köster"
+__copyright__ = "Copyright 2024, Johannes Köster"
+__email__ = "johannes.koester@uni-due.de"
+__license__ = "MIT"
+
+import types
+from typing import Mapping
+from snakemake_interface_software_deployment_plugins.settings import (
+    SoftwareDeploymentSettingsBase,
+)
+
+from snakemake_interface_common.plugin_registry.attribute_types import (
+    AttributeKind,
+    AttributeMode,
+    AttributeType,
+)
+from snakemake_interface_software_deployment_plugins.registry.plugin import Plugin
+from snakemake_interface_common.plugin_registry import PluginRegistryBase
+from snakemake_interface_software_deployment_plugins import EnvBase, _common as common
+
+
+class SoftwareDeploymentPluginRegistry(PluginRegistryBase):
+    """This class is a singleton that holds all registered executor plugins."""
+
+    @property
+    def module_prefix(self) -> str:
+        return common.software_deployment_plugin_module_prefix
+
+    def load_plugin(self, name: str, module: types.ModuleType) -> Plugin:
+        """Load a plugin by name."""
+        return Plugin(
+            _name=name,
+            _software_deployment_settings_cls=getattr(
+                module, "SoftwareDeploymentSettings", None
+            ),
+            _env_cls=module.EnvBase,
+        )
+
+    def expected_attributes(self) -> Mapping[str, AttributeType]:
+        return {
+            "SoftwareDeploymentSettings": AttributeType(
+                cls=SoftwareDeploymentSettingsBase,
+                mode=AttributeMode.OPTIONAL,
+                kind=AttributeKind.CLASS,
+            ),
+            "Env": AttributeType(
+                cls=EnvBase,
+                mode=AttributeMode.REQUIRED,
+                kind=AttributeKind.CLASS,
+            ),
+        }

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/registry/plugin.py

@@ -0,0 +1,46 @@
+__author__ = "Johannes Köster"
+__copyright__ = "Copyright 2024, Johannes Köster"
+__email__ = "johannes.koester@uni-due.de"
+__license__ = "MIT"
+
+from dataclasses import dataclass
+from typing import Optional, Type
+from snakemake_interface_software_deployment_plugins import EnvBase, EnvSpecBase
+from snakemake_interface_software_deployment_plugins.settings import (
+    SoftwareDeploymentSettingsBase,
+)
+import snakemake_interface_software_deployment_plugins._common as common
+
+from snakemake_interface_common.plugin_registry.plugin import PluginBase
+
+
+@dataclass
+class Plugin(PluginBase):
+    _software_deployment_settings_cls: Optional[Type[SoftwareDeploymentSettingsBase]]
+    _env_cls: Type[EnvBase]
+    _env_spec_cls: Type[EnvSpecBase]
+    _name: str
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def cli_prefix(self):
+        return "sdm-" + self._only_name
+
+    @property
+    def _only_name(self):
+        return self.name.replace(common.software_deployment_plugin_module_prefix, "")
+
+    @property
+    def settings_cls(self):
+        return self._software_deployment_settings_cls
+
+    @property
+    def env_cls(self):
+        return self._env_cls
+
+    @property
+    def env_spec_cls(self):
+        return self._env_spec_cls

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/settings.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+
+
+import snakemake_interface_common.plugin_registry.plugin
+
+
+@dataclass
+class SoftwareDeploymentSettingsBase(
+    snakemake_interface_common.plugin_registry.plugin.SettingsBase
+):
+    """Base class for software deployment settings.
+
+    Software deployment plugins can define a subclass of this class,
+    named 'SoftwareDeploymentProviderSettings'.
+    """
+
+    pass

snakemake_interface_software_deployment_plugins-0.1.0/snakemake_interface_software_deployment_plugins/tests.py

@@ -0,0 +1,95 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Type
+import subprocess as sp
+
+import pytest
+
+from snakemake_interface_software_deployment_plugins import (
+    ArchiveableEnvBase,
+    DeployableEnvBase,
+    EnvBase,
+    EnvSpecBase,
+)
+from snakemake_interface_software_deployment_plugins.settings import (
+    SoftwareDeploymentSettingsBase,
+)
+
+
+_TEST_SDM_NAME = "test-sdm"
+
+
+class TestSoftwareDeploymentBase(ABC):
+    __test__ = False
+
+    @abstractmethod
+    def get_env_spec(self) -> EnvSpecBase:
+        """
+        If the software deployment provider does not support deployable environments,
+        this method should return args for an existing environment spec that can be used
+        for testing
+        """
+        ...
+
+    @abstractmethod
+    def get_env_cls(self) -> Type[EnvBase]:
+        """
+        Return the environment class that should be tested.
+        """
+        ...
+
+    @abstractmethod
+    def get_test_cmd(self) -> str:
+        """
+        Return a test command that should be executed within the environment
+        with exit code 0 (i.e. without error).
+        """
+        ...
+
+    def get_software_deployment_provider_settings(
+        self,
+    ) -> Optional[SoftwareDeploymentSettingsBase]:
+        return None
+
+    def test_shellcmd(self, tmp_path):
+        env = self._get_env(tmp_path)
+
+        if isinstance(env, DeployableEnvBase):
+            pytest.skip("Environment is deployable, using test_deploy instead.")
+
+        cmd = self.get_test_cmd()
+        decorated_cmd = env.managed_decorate_shellcmd(cmd)
+        assert cmd != decorated_cmd
+        assert sp.run(decorated_cmd, shell=True).returncode == 0
+
+    def test_deploy(self, tmp_path):
+        env = self._get_env(tmp_path)
+        self._deploy(env, tmp_path)
+        cmd = env.managed_decorate_shellcmd(self.get_test_cmd())
+        assert sp.run(cmd, shell=True).returncode == 0
+
+    def test_archive(self, tmp_path):
+        env = self._get_env(tmp_path)
+        if not isinstance(env, ArchiveableEnvBase):
+            pytest.skip("Environment either not deployable or not archiveable.")
+
+        self._deploy(env, tmp_path)
+
+        env.archive()
+        assert any((tmp_path / "{_TEST_SDM_NAME}-archive").iterdir())
+
+    def _get_env(self, tmp_path) -> EnvBase:
+        env_cls = self.get_env_cls()
+        spec = self.get_env_spec()
+        args = {"settings": self.get_software_deployment_provider_settings()}
+        if issubclass(env_cls, DeployableEnvBase):
+            args["deployment_prefix"] = tmp_path / "deployments"
+        if issubclass(env_cls, ArchiveableEnvBase):
+            args["archive_prefix"] = tmp_path / "archives"
+        return env_cls(spec=spec, within=None, **args)
+
+    def _deploy(self, env: DeployableEnvBase, tmp_path):
+        if not isinstance(env, DeployableEnvBase):
+            pytest.skip("Environment is not deployable.")
+
+        env.deploy()
+        assert any((tmp_path / _TEST_SDM_NAME).iterdir())