snk-cli 0.1.4__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {snk_cli-0.1.4 → snk_cli-0.2.0}/.github/workflows/tests.yml +1 -1
- {snk_cli-0.1.4 → snk_cli-0.2.0}/PKG-INFO +2 -2
- {snk_cli-0.1.4 → snk_cli-0.2.0}/pyproject.toml +5 -10
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/__about__.py +1 -1
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/cli.py +14 -2
- snk_cli-0.2.0/src/snk_cli/conda.py +106 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/config/config.py +3 -3
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/config/utils.py +5 -1
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/dynamic_typer.py +1 -1
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/env.py +12 -29
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/run.py +33 -13
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/script.py +3 -10
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_subcommands.py +1 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_workflow_cli.py +2 -3
- snk_cli-0.2.0/tests/test_conda_env.py +11 -0
- snk_cli-0.1.4/.github/workflows/mkdocs.yml +0 -23
- snk_cli-0.1.4/src/snk_cli/subcommands/utils.py +0 -175
- {snk_cli-0.1.4 → snk_cli-0.2.0}/.github/workflows/publish.yml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/.gitignore +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/LICENSE.txt +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/README.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/index.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/cli.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/config.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/dynamic_typer.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/options.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/subcommands.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/testing.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/utils.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/validate.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/docs/reference/workflow.md +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/mkdocs.yml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/config/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/options/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/options/option.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/options/utils.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/config.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/subcommands/profile.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/testing.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/utils.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/validate.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/src/snk_cli/workflow.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/conftest.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/artic_v4.1.bed +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/config.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/cov.fasta +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/print_config/Snakefile +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/print_config/cli.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/print_config/config.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/print_config/snk.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/cli.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/config.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/resources/data.txt +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/snk.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/things/__about__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/workflow/Snakefile +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/workflow/envs/python.yml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/workflow/profiles/base/config.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/workflow/profiles/slurm/config.yaml +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/data/workflow/workflow/scripts/hello.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/__init__.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_dynamic_options.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_run.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_snk_config.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_cli/test_validate.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/test_dynamic_typer.py +0 -0
- {snk_cli-0.1.4 → snk_cli-0.2.0}/tests/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: snk-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Project-URL: Documentation, https://github.com/unknown/snk-cli#readme
|
|
5
5
|
Project-URL: Issues, https://github.com/unknown/snk-cli/issues
|
|
6
6
|
Project-URL: Source, https://github.com/unknown/snk-cli
|
|
@@ -22,7 +22,7 @@ Requires-Dist: datrie>=0.8.2
|
|
|
22
22
|
Requires-Dist: gitpython~=3.1
|
|
23
23
|
Requires-Dist: makefun~=1.15
|
|
24
24
|
Requires-Dist: pulp<2.8
|
|
25
|
-
Requires-Dist: snakemake
|
|
25
|
+
Requires-Dist: snakemake>=7
|
|
26
26
|
Requires-Dist: typer[all]~=0.9.0
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
|
|
@@ -25,7 +25,7 @@ classifiers = [
|
|
|
25
25
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
26
26
|
]
|
|
27
27
|
dependencies = [
|
|
28
|
-
"snakemake>=7
|
|
28
|
+
"snakemake>=7",
|
|
29
29
|
"typer[all]~=0.9.0",
|
|
30
30
|
"GitPython~=3.1",
|
|
31
31
|
"pulp<2.8", # Pin pulp <2.8 for snakemake: https://github.com/snakemake/snakemake/issues/2607
|
|
@@ -43,15 +43,10 @@ Source = "https://github.com/unknown/snk-cli"
|
|
|
43
43
|
path = "src/snk_cli/__about__.py"
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
[[tool.hatch.envs.
|
|
47
|
-
|
|
48
|
-
snakemake = ["7.32.4"]
|
|
46
|
+
[[tool.hatch.envs.snakemake.matrix]]
|
|
47
|
+
snakemake = ["7.32.4", "8.10.8"]
|
|
49
48
|
|
|
50
|
-
|
|
51
|
-
# python = ["3.11", "3.12"]
|
|
52
|
-
# snakemake = ["8.5.5"]
|
|
53
|
-
|
|
54
|
-
[tool.hatch.envs.test]
|
|
49
|
+
[tool.hatch.snakemake.test]
|
|
55
50
|
dependencies = [
|
|
56
51
|
"coverage[toml]>=6.5",
|
|
57
52
|
"pytest",
|
|
@@ -75,7 +70,7 @@ cov = [
|
|
|
75
70
|
]
|
|
76
71
|
|
|
77
72
|
[[tool.hatch.envs.all.matrix]]
|
|
78
|
-
python = ["3.
|
|
73
|
+
python = ["3.9", "3.10", "3.11", "3.12"]
|
|
79
74
|
|
|
80
75
|
[tool.hatch.envs.types]
|
|
81
76
|
dependencies = [
|
|
@@ -6,8 +6,6 @@ from pathlib import Path
|
|
|
6
6
|
from typing import Optional
|
|
7
7
|
import os
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
from snakemake import SNAKEFILE_CHOICES
|
|
11
9
|
from art import text2art
|
|
12
10
|
|
|
13
11
|
from snk_cli.dynamic_typer import DynamicTyper
|
|
@@ -224,6 +222,17 @@ class CLI(DynamicTyper):
|
|
|
224
222
|
Examples:
|
|
225
223
|
>>> CLI._find_snakefile()
|
|
226
224
|
"""
|
|
225
|
+
SNAKEFILE_CHOICES = list(
|
|
226
|
+
map(
|
|
227
|
+
Path,
|
|
228
|
+
(
|
|
229
|
+
"Snakefile",
|
|
230
|
+
"snakefile",
|
|
231
|
+
"workflow/Snakefile",
|
|
232
|
+
"workflow/snakefile",
|
|
233
|
+
),
|
|
234
|
+
)
|
|
235
|
+
)
|
|
227
236
|
for path in SNAKEFILE_CHOICES:
|
|
228
237
|
if (self.workflow.path / path).exists():
|
|
229
238
|
return self.workflow.path / path
|
|
@@ -244,6 +253,9 @@ class CLI(DynamicTyper):
|
|
|
244
253
|
info_dict = {}
|
|
245
254
|
info_dict["name"] = self.workflow.path.name
|
|
246
255
|
info_dict["version"] = self.version
|
|
256
|
+
info_dict["snakefile"] = str(self.snakefile)
|
|
257
|
+
info_dict["conda_prefix_dir"] = str(self.conda_prefix_dir)
|
|
258
|
+
info_dict["singularity_prefix_dir"] = str(self.singularity_prefix_dir)
|
|
247
259
|
info_dict["workflow_dir_path"] = str(self.workflow.path)
|
|
248
260
|
typer.echo(json.dumps(info_dict, indent=2))
|
|
249
261
|
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# This file contains functions to create and manage conda environments for snakemake workflows
|
|
2
|
+
# it needs to work with v 7 and 8 of snakemake
|
|
3
|
+
#
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from packaging import version
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
from snk_cli.utils import check_command_available
|
|
10
|
+
from snakemake.deployment.conda import Env
|
|
11
|
+
from snakemake.persistence import Persistence
|
|
12
|
+
import snakemake
|
|
13
|
+
|
|
14
|
+
snakemake_version = version.parse(snakemake.__version__)
|
|
15
|
+
is_snakemake_version_8_or_above = snakemake_version >= version.parse('8')
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class PersistenceMock(Persistence):
|
|
19
|
+
"""
|
|
20
|
+
Mock for workflow.persistence
|
|
21
|
+
"""
|
|
22
|
+
conda_env_path: Path = None
|
|
23
|
+
_metadata_path: Path = None
|
|
24
|
+
_incomplete_path: Path = None
|
|
25
|
+
shadow_path: Path = None
|
|
26
|
+
conda_env_archive_path: Path = None
|
|
27
|
+
container_img_path: Path = None
|
|
28
|
+
aux_path: Path = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_frontend():
|
|
32
|
+
if check_command_available("mamba"):
|
|
33
|
+
conda_frontend = "mamba"
|
|
34
|
+
else:
|
|
35
|
+
conda_frontend = "conda"
|
|
36
|
+
return conda_frontend
|
|
37
|
+
|
|
38
|
+
def create_workflow_v7(conda_prefix):
|
|
39
|
+
from snakemake.workflow import Workflow
|
|
40
|
+
|
|
41
|
+
conda_frontend = get_frontend()
|
|
42
|
+
workflow = Workflow(
|
|
43
|
+
snakefile=Path(),
|
|
44
|
+
overwrite_config=dict(),
|
|
45
|
+
overwrite_workdir=None,
|
|
46
|
+
overwrite_configfiles=[],
|
|
47
|
+
overwrite_clusterconfig=dict(),
|
|
48
|
+
conda_frontend=conda_frontend,
|
|
49
|
+
use_conda=True,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
persistence = PersistenceMock(
|
|
53
|
+
conda_env_path=Path(conda_prefix).resolve() if conda_prefix else None,
|
|
54
|
+
conda_env_archive_path=os.path.join(Path(".snakemake"), "conda-archive"),
|
|
55
|
+
)
|
|
56
|
+
if hasattr(workflow, "_persistence"):
|
|
57
|
+
workflow._persistence = persistence
|
|
58
|
+
else:
|
|
59
|
+
workflow.persistence = persistence
|
|
60
|
+
return workflow
|
|
61
|
+
|
|
62
|
+
def create_workflow_v8(
|
|
63
|
+
conda_prefix
|
|
64
|
+
):
|
|
65
|
+
from snakemake.api import Workflow
|
|
66
|
+
from snakemake.settings import (
|
|
67
|
+
ConfigSettings,
|
|
68
|
+
DeploymentSettings,
|
|
69
|
+
ResourceSettings,
|
|
70
|
+
WorkflowSettings,
|
|
71
|
+
StorageSettings,
|
|
72
|
+
)
|
|
73
|
+
conda_frontend = get_frontend()
|
|
74
|
+
workflow = Workflow(
|
|
75
|
+
config_settings=ConfigSettings(),
|
|
76
|
+
resource_settings=ResourceSettings(),
|
|
77
|
+
workflow_settings=WorkflowSettings(),
|
|
78
|
+
storage_settings=StorageSettings(),
|
|
79
|
+
deployment_settings=DeploymentSettings(
|
|
80
|
+
conda_frontend=conda_frontend,
|
|
81
|
+
conda_prefix=conda_prefix
|
|
82
|
+
),
|
|
83
|
+
)
|
|
84
|
+
persistence = PersistenceMock(
|
|
85
|
+
conda_env_path=Path(conda_prefix).resolve() if conda_prefix else None,
|
|
86
|
+
conda_env_archive_path=os.path.join(Path(".snakemake"), "conda-archive"),
|
|
87
|
+
)
|
|
88
|
+
if hasattr(workflow, "_persistence"):
|
|
89
|
+
workflow._persistence = persistence
|
|
90
|
+
else:
|
|
91
|
+
workflow.persistence = persistence
|
|
92
|
+
return workflow
|
|
93
|
+
|
|
94
|
+
def conda_environment_factory(env_file_path: Path, conda_prefix_dir_path: Path) -> Env:
|
|
95
|
+
"""
|
|
96
|
+
Create a snakemake environment object from a given environment file and conda prefix directory
|
|
97
|
+
"""
|
|
98
|
+
if is_snakemake_version_8_or_above:
|
|
99
|
+
snakemake_workflow = create_workflow_v8(
|
|
100
|
+
conda_prefix_dir_path
|
|
101
|
+
)
|
|
102
|
+
else:
|
|
103
|
+
snakemake_workflow = create_workflow_v7(conda_prefix_dir_path)
|
|
104
|
+
env_file_path = Path(env_file_path).resolve()
|
|
105
|
+
env = Env(snakemake_workflow, env_file=env_file_path)
|
|
106
|
+
return env
|
|
@@ -2,7 +2,7 @@ from pathlib import Path
|
|
|
2
2
|
from typing import List, Optional
|
|
3
3
|
import snakemake
|
|
4
4
|
from dataclasses import dataclass, field
|
|
5
|
-
from .utils import get_version_from_config
|
|
5
|
+
from .utils import get_version_from_config, load_configfile
|
|
6
6
|
import yaml
|
|
7
7
|
|
|
8
8
|
|
|
@@ -97,7 +97,7 @@ class SnkConfig:
|
|
|
97
97
|
if snk_config_path.stat().st_size == 0:
|
|
98
98
|
raise InvalidSnkConfigError(f"SNK config file is empty: {snk_config_path}") from ValueError
|
|
99
99
|
|
|
100
|
-
snk_config_dict =
|
|
100
|
+
snk_config_dict = load_configfile(snk_config_path)
|
|
101
101
|
snk_config_dict["version"] = get_version_from_config(snk_config_path, snk_config_dict)
|
|
102
102
|
if "annotations" in snk_config_dict:
|
|
103
103
|
# TODO: remove annotations in the future
|
|
@@ -257,4 +257,4 @@ def load_workflow_snakemake_config(workflow_dir_path: Path):
|
|
|
257
257
|
workflow_config_path = get_config_from_workflow_dir(workflow_dir_path)
|
|
258
258
|
if not workflow_config_path or not workflow_config_path.exists():
|
|
259
259
|
return {}
|
|
260
|
-
return
|
|
260
|
+
return load_configfile(workflow_config_path)
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
-
|
|
2
|
+
try:
|
|
3
|
+
# snakemake < 8.0.0
|
|
4
|
+
from snakemake import load_configfile
|
|
5
|
+
except ImportError:
|
|
6
|
+
from snakemake.common.configfile import _load_configfile as load_configfile # noqa: F401
|
|
3
7
|
|
|
4
8
|
def get_version_from_config(config_path: Path, config_dict: dict = None) -> str:
|
|
5
9
|
"""
|
|
@@ -217,8 +217,8 @@ class DynamicTyper:
|
|
|
217
217
|
flat_config = None
|
|
218
218
|
|
|
219
219
|
if kwargs.get("configfile"):
|
|
220
|
-
from snakemake import load_configfile
|
|
221
220
|
from .utils import flatten
|
|
221
|
+
from snk_cli.config.utils import load_configfile
|
|
222
222
|
|
|
223
223
|
snakemake_config = load_configfile(kwargs["configfile"])
|
|
224
224
|
flat_config = flatten(snakemake_config)
|
|
@@ -7,7 +7,7 @@ from typing import List, Optional
|
|
|
7
7
|
import typer
|
|
8
8
|
|
|
9
9
|
from snk_cli.dynamic_typer import DynamicTyper
|
|
10
|
-
from .
|
|
10
|
+
from snk_cli.conda import conda_environment_factory
|
|
11
11
|
from ..workflow import Workflow
|
|
12
12
|
from rich.console import Console
|
|
13
13
|
from rich.syntax import Syntax
|
|
@@ -22,22 +22,15 @@ def get_num_cores(default=4):
|
|
|
22
22
|
except:
|
|
23
23
|
return default
|
|
24
24
|
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
25
|
+
def create_conda_environment_wrapper(args):
|
|
26
|
+
"""
|
|
27
|
+
This wrapper is designed to be submitted to a ProcessPoolExecutor
|
|
28
|
+
"""
|
|
29
|
+
env_path_str, conda_prefix_dir_str = args
|
|
29
30
|
conda_prefix_dir = Path(conda_prefix_dir_str).resolve()
|
|
30
|
-
snakemake_workflow = create_snakemake_workflow(
|
|
31
|
-
snakefile,
|
|
32
|
-
config=snakemake_config,
|
|
33
|
-
configfiles=configfiles,
|
|
34
|
-
use_conda=True,
|
|
35
|
-
conda_prefix=conda_prefix_dir,
|
|
36
|
-
)
|
|
37
31
|
env_path = Path(env_path_str).resolve()
|
|
38
|
-
env = Env(snakemake_workflow, env_file=env_path)
|
|
39
32
|
try:
|
|
40
|
-
|
|
33
|
+
conda_environment_factory(env_path, conda_prefix_dir).create()
|
|
41
34
|
except CreateCondaEnvironmentException as e:
|
|
42
35
|
typer.secho(str(e), fg="red", err=True)
|
|
43
36
|
return 1
|
|
@@ -57,13 +50,6 @@ class EnvApp(DynamicTyper):
|
|
|
57
50
|
self.snakemake_config = snakemake_config
|
|
58
51
|
self.snakefile = snakefile
|
|
59
52
|
self.configfile = get_config_from_workflow_dir(self.workflow.path)
|
|
60
|
-
self.snakemake_workflow = create_snakemake_workflow(
|
|
61
|
-
self.snakefile,
|
|
62
|
-
config=self.snakemake_config,
|
|
63
|
-
configfiles=[self.configfile] if self.configfile else None,
|
|
64
|
-
use_conda=True,
|
|
65
|
-
conda_prefix=self.conda_prefix_dir.resolve(),
|
|
66
|
-
)
|
|
67
53
|
self.register_command(self.list, help="List the environments in the workflow.")
|
|
68
54
|
self.register_command(self.show, help="Show the contents of an environment.")
|
|
69
55
|
self.register_command(
|
|
@@ -86,7 +72,7 @@ class EnvApp(DynamicTyper):
|
|
|
86
72
|
|
|
87
73
|
table = Table("Name", "CMD", "Env", show_header=True, show_lines=True)
|
|
88
74
|
for env in self.workflow.environments:
|
|
89
|
-
conda =
|
|
75
|
+
conda = conda_environment_factory(env, self.conda_prefix_dir)
|
|
90
76
|
# address relative to cwd
|
|
91
77
|
address = Path(conda.address)
|
|
92
78
|
if address.exists():
|
|
@@ -137,7 +123,7 @@ class EnvApp(DynamicTyper):
|
|
|
137
123
|
cmd: List[str] = typer.Argument(..., help="The command to run in environment."),
|
|
138
124
|
):
|
|
139
125
|
env_path = self._get_conda_env_path(name)
|
|
140
|
-
env =
|
|
126
|
+
env = conda_environment_factory(env_path, self.conda_prefix_dir)
|
|
141
127
|
env.create()
|
|
142
128
|
cmd = self._shellcmd(env.address, " ".join(cmd))
|
|
143
129
|
if verbose:
|
|
@@ -154,7 +140,7 @@ class EnvApp(DynamicTyper):
|
|
|
154
140
|
):
|
|
155
141
|
if name:
|
|
156
142
|
env_path = self._get_conda_env_path(name)
|
|
157
|
-
env =
|
|
143
|
+
env = conda_environment_factory(env_path, self.conda_prefix_dir)
|
|
158
144
|
path = Path(env.address)
|
|
159
145
|
if not path.exists():
|
|
160
146
|
self.error(f"Environment {name} not created!")
|
|
@@ -176,15 +162,12 @@ class EnvApp(DynamicTyper):
|
|
|
176
162
|
env_args = [
|
|
177
163
|
(
|
|
178
164
|
path,
|
|
179
|
-
self.snakefile,
|
|
180
|
-
self.snakemake_config,
|
|
181
|
-
[self.configfile] if self.configfile else None,
|
|
182
165
|
self.conda_prefix_dir.resolve(),
|
|
183
166
|
)
|
|
184
167
|
for path in env_paths
|
|
185
168
|
]
|
|
186
169
|
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
187
|
-
status_codes = executor.map(
|
|
170
|
+
status_codes = executor.map(create_conda_environment_wrapper, env_args)
|
|
188
171
|
if any(status_codes):
|
|
189
172
|
self.error("Failed to create all conda environments!")
|
|
190
173
|
if names:
|
|
@@ -201,7 +184,7 @@ class EnvApp(DynamicTyper):
|
|
|
201
184
|
):
|
|
202
185
|
env_path = self._get_conda_env_path(name)
|
|
203
186
|
self.log(f"Activating {name} environment... (type 'exit' to deactivate)")
|
|
204
|
-
env =
|
|
187
|
+
env = conda_environment_factory(env_path, self.conda_prefix_dir)
|
|
205
188
|
env.create()
|
|
206
189
|
user_shell = os.environ.get("SHELL", "/bin/bash")
|
|
207
190
|
activate_cmd = self._shellcmd(env.address, user_shell)
|
|
@@ -5,6 +5,7 @@ from contextlib import contextmanager
|
|
|
5
5
|
|
|
6
6
|
from snk_cli.dynamic_typer import DynamicTyper
|
|
7
7
|
from snk_cli.options.option import Option
|
|
8
|
+
from snk_cli.conda import is_snakemake_version_8_or_above
|
|
8
9
|
from ..workflow import Workflow
|
|
9
10
|
from snk_cli.utils import (
|
|
10
11
|
parse_config_args,
|
|
@@ -167,7 +168,6 @@ class RunApp(DynamicTyper):
|
|
|
167
168
|
Examples:
|
|
168
169
|
>>> RunApp.run(target='my_target', configfile=Path('/path/to/config.yaml'), resource=[Path('/path/to/resource')], verbose=True)
|
|
169
170
|
"""
|
|
170
|
-
import snakemake
|
|
171
171
|
import shutil
|
|
172
172
|
import sys
|
|
173
173
|
|
|
@@ -274,8 +274,7 @@ class RunApp(DynamicTyper):
|
|
|
274
274
|
if dag:
|
|
275
275
|
return self._save_dag(snakemake_args=args, filename=dag)
|
|
276
276
|
try:
|
|
277
|
-
|
|
278
|
-
snakemake.main(args)
|
|
277
|
+
execute_snakemake(args)
|
|
279
278
|
except SystemExit as e:
|
|
280
279
|
status = int(str(e))
|
|
281
280
|
if status:
|
|
@@ -286,7 +285,6 @@ class RunApp(DynamicTyper):
|
|
|
286
285
|
|
|
287
286
|
def _save_dag(self, snakemake_args: List[str], filename: Path):
|
|
288
287
|
from contextlib import redirect_stdout
|
|
289
|
-
import snakemake
|
|
290
288
|
import subprocess
|
|
291
289
|
import io
|
|
292
290
|
|
|
@@ -300,8 +298,7 @@ class RunApp(DynamicTyper):
|
|
|
300
298
|
with redirect_stdout(snakemake_output):
|
|
301
299
|
# Capture the output of snakemake.main(args) using a try-except block
|
|
302
300
|
try:
|
|
303
|
-
|
|
304
|
-
snakemake.main(snakemake_args)
|
|
301
|
+
execute_snakemake(snakemake_args)
|
|
305
302
|
except SystemExit: # Catch SystemExit exception to prevent termination
|
|
306
303
|
pass
|
|
307
304
|
try:
|
|
@@ -415,6 +412,24 @@ class RunApp(DynamicTyper):
|
|
|
415
412
|
)
|
|
416
413
|
remove_resource(copied_resource)
|
|
417
414
|
|
|
415
|
+
def execute_snakemake(args):
|
|
416
|
+
"""
|
|
417
|
+
Execute snakemake with the given arguments.
|
|
418
|
+
|
|
419
|
+
Args:
|
|
420
|
+
args: The arguments to pass to snakemake.
|
|
421
|
+
|
|
422
|
+
Side Effects:
|
|
423
|
+
Executes snakemake with the given arguments.
|
|
424
|
+
"""
|
|
425
|
+
import snakemake
|
|
426
|
+
if is_snakemake_version_8_or_above:
|
|
427
|
+
from snakemake import cli
|
|
428
|
+
cli.parse_config = parse_config_monkeypatch
|
|
429
|
+
cli.main(args)
|
|
430
|
+
else:
|
|
431
|
+
snakemake.parse_config = parse_config_monkeypatch
|
|
432
|
+
snakemake.main(args)
|
|
418
433
|
|
|
419
434
|
def parse_config_monkeypatch(args):
|
|
420
435
|
"""
|
|
@@ -427,8 +442,13 @@ def parse_config_monkeypatch(args):
|
|
|
427
442
|
dict: The parsed config.
|
|
428
443
|
"""
|
|
429
444
|
import yaml
|
|
430
|
-
import snakemake
|
|
431
445
|
import re
|
|
446
|
+
if is_snakemake_version_8_or_above:
|
|
447
|
+
from snakemake.cli import parse_key_value_arg, update_config, _bool_parser
|
|
448
|
+
entries = args
|
|
449
|
+
else:
|
|
450
|
+
from snakemake import parse_key_value_arg, update_config, _bool_parser
|
|
451
|
+
entries = args.config
|
|
432
452
|
|
|
433
453
|
class NoDatesSafeLoader(yaml.SafeLoader):
|
|
434
454
|
@classmethod
|
|
@@ -465,12 +485,12 @@ def parse_config_monkeypatch(args):
|
|
|
465
485
|
s = s.replace(": None", ": null")
|
|
466
486
|
return yaml.load(s, Loader=NoDatesSafeLoader)
|
|
467
487
|
|
|
468
|
-
parsers = [int, float,
|
|
488
|
+
parsers = [int, float, _bool_parser, _yaml_safe_load, str]
|
|
469
489
|
config = dict()
|
|
470
|
-
if
|
|
490
|
+
if entries is not None:
|
|
471
491
|
valid = re.compile(r"[a-zA-Z_]\w*$")
|
|
472
|
-
for entry in
|
|
473
|
-
key, val =
|
|
492
|
+
for entry in entries:
|
|
493
|
+
key, val = parse_key_value_arg(
|
|
474
494
|
entry,
|
|
475
495
|
errmsg="Invalid config definition: Config entries have to be defined as name=value pairs.",
|
|
476
496
|
)
|
|
@@ -480,7 +500,7 @@ def parse_config_monkeypatch(args):
|
|
|
480
500
|
)
|
|
481
501
|
v = None
|
|
482
502
|
if val == "" or val == "None":
|
|
483
|
-
|
|
503
|
+
update_config(config, {key: v})
|
|
484
504
|
continue
|
|
485
505
|
for parser in parsers:
|
|
486
506
|
try:
|
|
@@ -491,5 +511,5 @@ def parse_config_monkeypatch(args):
|
|
|
491
511
|
except:
|
|
492
512
|
pass
|
|
493
513
|
assert v is not None
|
|
494
|
-
|
|
514
|
+
update_config(config, {key: v})
|
|
495
515
|
return config
|
|
@@ -6,11 +6,11 @@ from typing import List
|
|
|
6
6
|
import typer
|
|
7
7
|
|
|
8
8
|
from ..dynamic_typer import DynamicTyper
|
|
9
|
-
from .
|
|
9
|
+
from snk_cli.conda import conda_environment_factory
|
|
10
10
|
from ..workflow import Workflow
|
|
11
11
|
from rich.console import Console
|
|
12
12
|
from rich.syntax import Syntax
|
|
13
|
-
from snakemake.deployment.conda import Conda
|
|
13
|
+
from snakemake.deployment.conda import Conda
|
|
14
14
|
from snk_cli.config.config import get_config_from_workflow_dir
|
|
15
15
|
|
|
16
16
|
|
|
@@ -116,14 +116,7 @@ class ScriptApp(DynamicTyper):
|
|
|
116
116
|
cmd = [executor, f'"{script_path}"'] + args
|
|
117
117
|
if env:
|
|
118
118
|
env_path = self._get_conda_env_path(env)
|
|
119
|
-
|
|
120
|
-
self.snakefile,
|
|
121
|
-
config=self.snakemake_config,
|
|
122
|
-
configfiles=[self.configfile] if self.configfile else None,
|
|
123
|
-
use_conda=True,
|
|
124
|
-
conda_prefix=self.conda_prefix_dir.resolve(),
|
|
125
|
-
)
|
|
126
|
-
env = Env(workflow, env_file=env_path.resolve())
|
|
119
|
+
env = conda_environment_factory(env_path, self.conda_prefix_dir)
|
|
127
120
|
env.create()
|
|
128
121
|
cmd = self._shellcmd(env.address, " ".join(cmd))
|
|
129
122
|
else:
|
|
@@ -13,6 +13,7 @@ import pytest
|
|
|
13
13
|
(["env", "run", "python", "which python"], ["bin/python"], []),
|
|
14
14
|
(["env", "activate", "python"], [], ["Activating python environment...", "Exiting python environment..."]),
|
|
15
15
|
(["env", "remove", "-f"], ["Deleted"], []),
|
|
16
|
+
(["info"], ["name", "version", "snakefile", "conda_prefix_dir", "singularity_prefix_dir", "workflow_dir_path"], []),
|
|
16
17
|
])
|
|
17
18
|
def test_snk_cli_command(capfd, local_runner, cmd, expected_in_stdout, expected_in_stderr):
|
|
18
19
|
res = local_runner(cmd)
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
from snk_cli.utils import flatten, convert_key_to_snakemake_format
|
|
3
|
-
import snakemake
|
|
4
3
|
import pytest
|
|
5
4
|
from ..utils import SnkCliRunner
|
|
6
|
-
|
|
5
|
+
from snk_cli.config.utils import load_configfile
|
|
7
6
|
|
|
8
7
|
def test_flatten(example_config: Path):
|
|
9
|
-
config =
|
|
8
|
+
config = load_configfile(example_config)
|
|
10
9
|
flat_config = flatten(config)
|
|
11
10
|
assert flat_config["diffexp:contrasts:A-vs-B"] == ["A", "B"]
|
|
12
11
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from snk_cli.conda import conda_environment_factory
|
|
2
|
+
from snakemake.deployment.conda import Env
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_conda_env(tmp_path):
|
|
7
|
+
env = conda_environment_factory("tests/data/workflow/workflow/envs/python.yml", tmp_path)
|
|
8
|
+
assert isinstance(env, Env)
|
|
9
|
+
assert not Path(env.address).exists()
|
|
10
|
+
env.create()
|
|
11
|
+
assert Path(env.address).exists()
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
name: mkdocs
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- master
|
|
7
|
-
- main
|
|
8
|
-
permissions:
|
|
9
|
-
contents: write
|
|
10
|
-
jobs:
|
|
11
|
-
deploy:
|
|
12
|
-
runs-on: ubuntu-latest
|
|
13
|
-
steps:
|
|
14
|
-
- uses: actions/checkout@v3
|
|
15
|
-
- uses: actions/setup-python@v4
|
|
16
|
-
with:
|
|
17
|
-
python-version: 3.x
|
|
18
|
-
- uses: actions/cache@v2
|
|
19
|
-
with:
|
|
20
|
-
key: ${{ github.ref }}
|
|
21
|
-
path: .cache
|
|
22
|
-
- run: pip install "mkdocstrings==0.22.0" "mkdocstrings-python==1.3.*" "mkdocs-material"
|
|
23
|
-
- run: mkdocs gh-deploy --force
|
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from snakemake import (
|
|
4
|
-
Workflow,
|
|
5
|
-
update_config,
|
|
6
|
-
load_configfile,
|
|
7
|
-
dict_to_key_value_args,
|
|
8
|
-
common,
|
|
9
|
-
)
|
|
10
|
-
from snakemake.persistence import Persistence
|
|
11
|
-
from ..utils import check_command_available
|
|
12
|
-
import os
|
|
13
|
-
|
|
14
|
-
@dataclass
|
|
15
|
-
class PersistenceMock(Persistence):
|
|
16
|
-
"""
|
|
17
|
-
Mock for workflow.persistence
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
conda_env_path: Path = None
|
|
21
|
-
_metadata_path: Path = None
|
|
22
|
-
_incomplete_path: Path = None
|
|
23
|
-
shadow_path: Path = None
|
|
24
|
-
conda_env_archive_path: Path = None
|
|
25
|
-
container_img_path: Path = None
|
|
26
|
-
aux_path: Path = None
|
|
27
|
-
|
|
28
|
-
def create_snakemake_workflow(
|
|
29
|
-
snakefile,
|
|
30
|
-
cache=None,
|
|
31
|
-
lint=None,
|
|
32
|
-
cores=1,
|
|
33
|
-
nodes=None,
|
|
34
|
-
max_threads=None,
|
|
35
|
-
resources=dict(),
|
|
36
|
-
overwrite_threads=None,
|
|
37
|
-
overwrite_scatter=None,
|
|
38
|
-
overwrite_resource_scopes=None,
|
|
39
|
-
default_resources=None,
|
|
40
|
-
overwrite_resources=None,
|
|
41
|
-
config=dict(),
|
|
42
|
-
configfiles=None,
|
|
43
|
-
config_args=None,
|
|
44
|
-
workdir=None,
|
|
45
|
-
printshellcmds=False,
|
|
46
|
-
rerun_triggers=["mtime", "params", "input", "software-env", "code"],
|
|
47
|
-
conda_cleanup_envs=False,
|
|
48
|
-
latency_wait=3,
|
|
49
|
-
print_compilation=False,
|
|
50
|
-
debug=False,
|
|
51
|
-
all_temp=False,
|
|
52
|
-
jobscript=None,
|
|
53
|
-
overwrite_shellcmd=None,
|
|
54
|
-
restart_times=0,
|
|
55
|
-
attempt=1,
|
|
56
|
-
verbose=False,
|
|
57
|
-
use_conda=False,
|
|
58
|
-
use_singularity=False,
|
|
59
|
-
use_env_modules=False,
|
|
60
|
-
singularity_args="",
|
|
61
|
-
conda_frontend="conda",
|
|
62
|
-
conda_prefix=None,
|
|
63
|
-
conda_cleanup_pkgs=None,
|
|
64
|
-
list_conda_envs=False,
|
|
65
|
-
singularity_prefix=None,
|
|
66
|
-
shadow_prefix=None,
|
|
67
|
-
scheduler="ilp",
|
|
68
|
-
scheduler_ilp_solver=None,
|
|
69
|
-
mode=common.Mode.default,
|
|
70
|
-
wrapper_prefix=None,
|
|
71
|
-
default_remote_prefix="",
|
|
72
|
-
assume_shared_fs=True,
|
|
73
|
-
keep_metadata=True,
|
|
74
|
-
edit_notebook=None,
|
|
75
|
-
envvars=None,
|
|
76
|
-
overwrite_groups=None,
|
|
77
|
-
group_components=None,
|
|
78
|
-
max_inventory_wait_time=20,
|
|
79
|
-
execute_subworkflows=True,
|
|
80
|
-
conda_not_block_search_path_envvars=False,
|
|
81
|
-
scheduler_solver_path=None,
|
|
82
|
-
conda_base_path=None,
|
|
83
|
-
local_groupid="local",
|
|
84
|
-
):
|
|
85
|
-
overwrite_config = dict()
|
|
86
|
-
if configfiles is None:
|
|
87
|
-
configfiles = []
|
|
88
|
-
for f in configfiles:
|
|
89
|
-
# get values to override. Later configfiles override earlier ones.
|
|
90
|
-
update_config(overwrite_config, load_configfile(f))
|
|
91
|
-
# convert provided paths to absolute paths
|
|
92
|
-
configfiles = list(map(os.path.abspath, configfiles))
|
|
93
|
-
|
|
94
|
-
# directly specified elements override any configfiles
|
|
95
|
-
if config:
|
|
96
|
-
update_config(overwrite_config, config)
|
|
97
|
-
if config_args is None:
|
|
98
|
-
config_args = dict_to_key_value_args(config)
|
|
99
|
-
|
|
100
|
-
if workdir:
|
|
101
|
-
if not os.path.exists(workdir):
|
|
102
|
-
os.makedirs(workdir)
|
|
103
|
-
workdir = os.path.abspath(workdir)
|
|
104
|
-
os.chdir(workdir)
|
|
105
|
-
|
|
106
|
-
if check_command_available("mamba"):
|
|
107
|
-
conda_frontend = "mamba"
|
|
108
|
-
|
|
109
|
-
workflow = Workflow(
|
|
110
|
-
snakefile=snakefile,
|
|
111
|
-
rerun_triggers=rerun_triggers,
|
|
112
|
-
jobscript=jobscript,
|
|
113
|
-
overwrite_shellcmd=overwrite_shellcmd,
|
|
114
|
-
overwrite_config=overwrite_config,
|
|
115
|
-
overwrite_workdir=workdir,
|
|
116
|
-
overwrite_configfiles=configfiles,
|
|
117
|
-
overwrite_clusterconfig=dict(),
|
|
118
|
-
overwrite_threads=overwrite_threads,
|
|
119
|
-
max_threads=max_threads,
|
|
120
|
-
overwrite_scatter=overwrite_scatter,
|
|
121
|
-
overwrite_groups=overwrite_groups,
|
|
122
|
-
overwrite_resources=overwrite_resources,
|
|
123
|
-
overwrite_resource_scopes=overwrite_resource_scopes,
|
|
124
|
-
group_components=group_components,
|
|
125
|
-
config_args=config_args,
|
|
126
|
-
debug=debug,
|
|
127
|
-
verbose=verbose,
|
|
128
|
-
use_conda=use_conda or list_conda_envs or conda_cleanup_envs,
|
|
129
|
-
use_singularity=use_singularity,
|
|
130
|
-
use_env_modules=use_env_modules,
|
|
131
|
-
conda_frontend=conda_frontend,
|
|
132
|
-
conda_prefix=conda_prefix,
|
|
133
|
-
conda_cleanup_pkgs=conda_cleanup_pkgs,
|
|
134
|
-
singularity_prefix=singularity_prefix,
|
|
135
|
-
shadow_prefix=shadow_prefix,
|
|
136
|
-
singularity_args=singularity_args,
|
|
137
|
-
scheduler_type=scheduler,
|
|
138
|
-
scheduler_ilp_solver=scheduler_ilp_solver,
|
|
139
|
-
mode=mode,
|
|
140
|
-
wrapper_prefix=wrapper_prefix,
|
|
141
|
-
printshellcmds=printshellcmds,
|
|
142
|
-
restart_times=restart_times,
|
|
143
|
-
attempt=attempt,
|
|
144
|
-
default_remote_provider=None,
|
|
145
|
-
default_remote_prefix=default_remote_prefix,
|
|
146
|
-
run_local=True,
|
|
147
|
-
assume_shared_fs=assume_shared_fs,
|
|
148
|
-
default_resources=default_resources,
|
|
149
|
-
cache=cache,
|
|
150
|
-
cores=cores,
|
|
151
|
-
nodes=nodes,
|
|
152
|
-
resources=resources,
|
|
153
|
-
edit_notebook=edit_notebook,
|
|
154
|
-
envvars=envvars,
|
|
155
|
-
max_inventory_wait_time=max_inventory_wait_time,
|
|
156
|
-
conda_not_block_search_path_envvars=conda_not_block_search_path_envvars,
|
|
157
|
-
execute_subworkflows=execute_subworkflows,
|
|
158
|
-
scheduler_solver_path=scheduler_solver_path,
|
|
159
|
-
conda_base_path=conda_base_path,
|
|
160
|
-
check_envvars=not lint, # for linting, we do not need to check whether requested envvars exist
|
|
161
|
-
all_temp=all_temp,
|
|
162
|
-
local_groupid=local_groupid,
|
|
163
|
-
keep_metadata=keep_metadata,
|
|
164
|
-
latency_wait=latency_wait,
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
persistence = PersistenceMock(
|
|
168
|
-
conda_env_path=Path(conda_prefix).resolve() if conda_prefix else None,
|
|
169
|
-
conda_env_archive_path=os.path.join(Path(".snakemake"), "conda-archive"),
|
|
170
|
-
)
|
|
171
|
-
if hasattr(workflow, "_persistence"):
|
|
172
|
-
workflow._persistence = persistence
|
|
173
|
-
else:
|
|
174
|
-
workflow.persistence = persistence
|
|
175
|
-
return workflow
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|