easylink 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- easylink/_version.py +1 -1
- easylink/configuration.py +5 -5
- easylink/graph_components.py +48 -51
- easylink/implementation.py +70 -10
- easylink/pipeline.py +127 -24
- easylink/pipeline_graph.py +46 -26
- easylink/pipeline_schema_constants/__init__.py +11 -7
- easylink/pipeline_schema_constants/development.py +2 -23
- easylink/pipeline_schema_constants/testing.py +243 -17
- easylink/rule.py +60 -140
- easylink/runner.py +14 -9
- easylink/step.py +397 -143
- easylink/utilities/spark.smk +2 -2
- easylink/utilities/splitter_utils.py +35 -0
- {easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/METADATA +22 -14
- {easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/RECORD +19 -19
- {easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/WHEEL +1 -1
- {easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/entry_points.txt +0 -0
- {easylink-0.1.13.dist-info → easylink-0.1.15.dist-info}/top_level.txt +0 -0
easylink/runner.py
CHANGED
@@ -19,6 +19,7 @@ from snakemake.cli import main as snake_main
|
|
19
19
|
|
20
20
|
from easylink.configuration import Config, load_params_from_specification
|
21
21
|
from easylink.pipeline import Pipeline
|
22
|
+
from easylink.pipeline_schema import PIPELINE_SCHEMAS, PipelineSchema
|
22
23
|
from easylink.utilities.data_utils import (
|
23
24
|
copy_configuration_files_to_results_directory,
|
24
25
|
create_results_directory,
|
@@ -30,11 +31,12 @@ from easylink.utilities.paths import EASYLINK_TEMP
|
|
30
31
|
|
31
32
|
def main(
|
32
33
|
command: str,
|
33
|
-
pipeline_specification: str,
|
34
|
-
input_data: str,
|
35
|
-
computing_environment: str | None,
|
36
|
-
results_dir: str,
|
34
|
+
pipeline_specification: str | Path,
|
35
|
+
input_data: str | Path,
|
36
|
+
computing_environment: str | Path | None,
|
37
|
+
results_dir: str | Path,
|
37
38
|
debug=False,
|
39
|
+
potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
|
38
40
|
) -> None:
|
39
41
|
"""Runs an EasyLink command.
|
40
42
|
|
@@ -44,8 +46,8 @@ def main(
|
|
44
46
|
files and then calls on `Snakemake <https://snakemake.readthedocs.io/en/stable/>`_
|
45
47
|
to act as the workflow manager.
|
46
48
|
|
47
|
-
|
48
|
-
|
49
|
+
Parameters
|
50
|
+
----------
|
49
51
|
command
|
50
52
|
The command to run. Current supported commands include "run" and "generate_dag".
|
51
53
|
pipeline_specification
|
@@ -61,11 +63,14 @@ def main(
|
|
61
63
|
debug
|
62
64
|
If False (the default), will suppress some of the workflow output. This
|
63
65
|
is intended to only be used for testing and development purposes.
|
66
|
+
potential_schemas
|
67
|
+
A list of potential schemas to validate the pipeline configuration against.
|
68
|
+
This is primarily used for testing purposes. Defaults to the supported schemas.
|
64
69
|
"""
|
65
70
|
config_params = load_params_from_specification(
|
66
71
|
pipeline_specification, input_data, computing_environment, results_dir
|
67
72
|
)
|
68
|
-
config = Config(config_params)
|
73
|
+
config = Config(config_params, potential_schemas)
|
69
74
|
pipeline = Pipeline(config)
|
70
75
|
# After validation is completed, create the results directory
|
71
76
|
create_results_directory(Path(results_dir))
|
@@ -84,7 +89,7 @@ def main(
|
|
84
89
|
environment_args = _get_environment_args(config)
|
85
90
|
singularity_args = _get_singularity_args(config)
|
86
91
|
# Set source cache in appropriate location to avoid jenkins failures
|
87
|
-
os.environ["XDG_CACHE_HOME"] = results_dir + "/.snakemake/source_cache"
|
92
|
+
os.environ["XDG_CACHE_HOME"] = str(results_dir) + "/.snakemake/source_cache"
|
88
93
|
# We need to set a dummy environment variable to avoid logging a wall of text.
|
89
94
|
# TODO [MIC-4920]: Remove when https://github.com/snakemake/snakemake-interface-executor-plugins/issues/55 merges
|
90
95
|
os.environ["foo"] = "bar"
|
@@ -92,7 +97,7 @@ def main(
|
|
92
97
|
"--snakefile",
|
93
98
|
str(snakefile),
|
94
99
|
"--directory",
|
95
|
-
results_dir,
|
100
|
+
str(results_dir),
|
96
101
|
"--cores",
|
97
102
|
"all",
|
98
103
|
"--jobs",
|