easylink 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {easylink-0.1.4 → easylink-0.1.6}/CHANGELOG.rst +9 -1
- {easylink-0.1.4 → easylink-0.1.6}/PKG-INFO +1 -1
- easylink-0.1.6/docs/source/api_reference/pipeline_schema_constants/testing.rst +1 -0
- easylink-0.1.6/src/easylink/_version.py +1 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/configuration.py +1 -1
- easylink-0.1.6/src/easylink/pipeline_schema_constants/__init__.py +12 -0
- easylink-0.1.4/src/easylink/pipeline_schema_constants/tests.py → easylink-0.1.6/src/easylink/pipeline_schema_constants/testing.py +90 -1
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/step.py +63 -21
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/PKG-INFO +1 -1
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/SOURCES.txt +29 -2
- easylink-0.1.6/tests/specifications/unit/environment_minimum.yaml +2 -0
- easylink-0.1.6/tests/specifications/unit/environment_spark_slurm.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_bad_combined_implementations.yaml +14 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_bad_implementation.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_bad_loop_formatting.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_bad_step.yaml +18 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_bad_type_key.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml +18 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_bad_topology.yaml +18 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_two_steps.yaml +16 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_extra_node.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_iteration.yaml +19 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml +19 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_missing_node.yaml +17 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_parallel.yaml +24 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_implementation_name.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_implementations.yaml +14 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_loop_nodes.yaml +14 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_step.yaml +4 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_substeps.yaml +17 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_missing_type_key.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_nested_templated_steps.yaml +21 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_out_of_order.yaml +15 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_spark.yaml +7 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_type_config_mismatch.yaml +18 -0
- easylink-0.1.6/tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml +19 -0
- easylink-0.1.6/tests/unit/conftest.py +157 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline_graph.py +146 -14
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_runner.py +5 -5
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_step.py +100 -30
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_validations.py +22 -9
- easylink-0.1.4/docs/source/api_reference/pipeline_schema_constants/tests.rst +0 -1
- easylink-0.1.4/src/easylink/_version.py +0 -1
- easylink-0.1.4/src/easylink/pipeline_schema_constants/__init__.py +0 -11
- easylink-0.1.4/tests/unit/conftest.py +0 -827
- {easylink-0.1.4 → easylink-0.1.6}/.bandit +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.flake8 +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.github/CODEOWNERS +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.github/pull_request_template.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/build.yml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/deploy.yml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/update_readme.yml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.gitignore +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/.readthedocs.yml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/Jenkinsfile +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/Makefile +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/README.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/Makefile +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/nitpick-exceptions +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/_static/style.css +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/_templates/layout.html +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/configuration.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/graph_components.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/implementation.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_graph.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema_constants/development.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema_constants/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/rule.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/runner.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/step.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/data_utils.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/general_utils.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/paths.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/validation_utils.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/concepts/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/conf.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/glossary.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/cli.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/tutorials/getting_started.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/tutorials/index.rst +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/pyproject.toml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/python_versions.json +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/pytype.cfg +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/setup.cfg +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/setup.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/__about__.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/__init__.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/cli.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/graph_components.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/images/spark_cluster/Dockerfile +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/images/spark_cluster/README.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/implementation.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/implementation_metadata.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_graph.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_schema.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_schema_constants/development.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/rule.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/runner.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/README.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/build-containers-local.sh +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/build-containers-remote.sh +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/create_input_files.ipynb +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_1.csv +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_1.parquet +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_2.csv +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_2.parquet +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/README.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/dummy_step.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/python_pandas.def +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/README.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/dummy_step.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/python_pyspark.def +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/README.md +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/dummy_step.R +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/r-image.def +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/test.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/__init__.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/data_utils.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/general_utils.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/paths.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/spark.smk +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/validation_utils.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/dependency_links.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/entry_points.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/not-zip-safe +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/requires.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/top_level.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/__init__.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/conftest.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/e2e/test_easylink_run.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/e2e/test_step_types.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake_slurm.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake_spark.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/environment_local.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/input_data.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/pipeline.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/environment_slurm.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/pipeline.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/pipeline_expanded.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/environment_spark_slurm.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/pipeline.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/pipeline_spark.yaml +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/__init__.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/implemented_rule_local.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/implemented_rule_slurm.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/pipeline_local.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/pipeline_slurm.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/target_rule.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/validation_rule.txt +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_cli.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_config.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_data_utils.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_general_utils.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_graph_components.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_implementation.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline_schema.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_rule.py +0 -0
- {easylink-0.1.4 → easylink-0.1.6}/update_readme.py +0 -0
@@ -1,4 +1,12 @@
|
|
1
|
-
**0.1.
|
1
|
+
**0.1.6 - 2/21/25**
|
2
|
+
|
3
|
+
- Move test dictionaries to yaml files
|
4
|
+
|
5
|
+
**0.1.5 - 2/20/25**
|
6
|
+
|
7
|
+
- Fix handling of templated steps when no looping or parallelism is requested
|
8
|
+
|
9
|
+
**0.1.4 - 2/20/25**
|
2
10
|
|
3
11
|
- Implement duplicate_template_step method on TemplatedStep class
|
4
12
|
|
@@ -0,0 +1 @@
|
|
1
|
+
.. automodule:: easylink.pipeline_schema_constants.testing
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "0.1.6"
|
@@ -97,7 +97,7 @@ class Config(LayeredConfigTree):
|
|
97
97
|
def __init__(
|
98
98
|
self,
|
99
99
|
config_params: dict[str, Any],
|
100
|
-
potential_schemas:
|
100
|
+
potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
|
101
101
|
) -> None:
|
102
102
|
super().__init__(layers=["initial_data", "default", "user_configured"])
|
103
103
|
self.update(DEFAULT_ENVIRONMENT, layer="default")
|
@@ -0,0 +1,12 @@
|
|
1
|
+
from easylink.pipeline_schema_constants import development, testing
|
2
|
+
|
3
|
+
ALLOWED_SCHEMA_PARAMS = {
|
4
|
+
"development": development.SCHEMA_PARAMS,
|
5
|
+
}
|
6
|
+
|
7
|
+
TESTING_SCHEMA_PARAMS = {
|
8
|
+
"integration": testing.SINGLE_STEP_SCHEMA_PARAMS,
|
9
|
+
"combine_bad_topology": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
|
10
|
+
"combine_bad_implementation_names": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
|
11
|
+
"nested_templated_steps": testing.NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS,
|
12
|
+
}
|
@@ -11,7 +11,14 @@ from easylink.graph_components import (
|
|
11
11
|
OutputSlot,
|
12
12
|
OutputSlotMapping,
|
13
13
|
)
|
14
|
-
from easylink.step import
|
14
|
+
from easylink.step import (
|
15
|
+
HierarchicalStep,
|
16
|
+
InputStep,
|
17
|
+
LoopStep,
|
18
|
+
OutputStep,
|
19
|
+
ParallelStep,
|
20
|
+
Step,
|
21
|
+
)
|
15
22
|
from easylink.utilities.validation_utils import validate_input_file_dummy
|
16
23
|
|
17
24
|
SINGLE_STEP_NODES = [
|
@@ -128,3 +135,85 @@ BAD_COMBINED_TOPOLOGY_NODES = [
|
|
128
135
|
]
|
129
136
|
|
130
137
|
BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS = (BAD_COMBINED_TOPOLOGY_NODES, SINGLE_STEP_EDGES)
|
138
|
+
|
139
|
+
|
140
|
+
NESTED_TEMPLATED_STEPS_NODES = [
|
141
|
+
InputStep(),
|
142
|
+
LoopStep(
|
143
|
+
template_step=ParallelStep(
|
144
|
+
template_step=HierarchicalStep(
|
145
|
+
step_name="step_1",
|
146
|
+
input_slots=[
|
147
|
+
InputSlot(
|
148
|
+
name="step_1_main_input",
|
149
|
+
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
|
150
|
+
validator=validate_input_file_dummy,
|
151
|
+
),
|
152
|
+
],
|
153
|
+
output_slots=[OutputSlot("step_1_main_output")],
|
154
|
+
nodes=[
|
155
|
+
Step(
|
156
|
+
step_name="step_1a",
|
157
|
+
input_slots=[
|
158
|
+
InputSlot(
|
159
|
+
name="step_1a_main_input",
|
160
|
+
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
|
161
|
+
validator=validate_input_file_dummy,
|
162
|
+
),
|
163
|
+
],
|
164
|
+
output_slots=[OutputSlot("step_1a_main_output")],
|
165
|
+
),
|
166
|
+
Step(
|
167
|
+
step_name="step_1b",
|
168
|
+
input_slots=[
|
169
|
+
InputSlot(
|
170
|
+
name="step_1b_main_input",
|
171
|
+
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
|
172
|
+
validator=validate_input_file_dummy,
|
173
|
+
),
|
174
|
+
],
|
175
|
+
output_slots=[OutputSlot("step_1b_main_output")],
|
176
|
+
),
|
177
|
+
],
|
178
|
+
edges=[
|
179
|
+
EdgeParams(
|
180
|
+
source_node="step_1a",
|
181
|
+
target_node="step_1b",
|
182
|
+
output_slot="step_1a_main_output",
|
183
|
+
input_slot="step_1b_main_input",
|
184
|
+
),
|
185
|
+
],
|
186
|
+
input_slot_mappings=[
|
187
|
+
InputSlotMapping(
|
188
|
+
parent_slot="step_1_main_input",
|
189
|
+
child_node="step_1a",
|
190
|
+
child_slot="step_1a_main_input",
|
191
|
+
),
|
192
|
+
],
|
193
|
+
output_slot_mappings=[
|
194
|
+
OutputSlotMapping(
|
195
|
+
parent_slot="step_1_main_output",
|
196
|
+
child_node="step_1b",
|
197
|
+
child_slot="step_1b_main_output",
|
198
|
+
),
|
199
|
+
],
|
200
|
+
),
|
201
|
+
),
|
202
|
+
self_edges=[
|
203
|
+
EdgeParams(
|
204
|
+
source_node="step_1",
|
205
|
+
target_node="step_1",
|
206
|
+
output_slot="step_1_main_output",
|
207
|
+
input_slot="step_1_main_input",
|
208
|
+
),
|
209
|
+
],
|
210
|
+
),
|
211
|
+
OutputStep(
|
212
|
+
input_slots=[
|
213
|
+
InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
|
214
|
+
],
|
215
|
+
),
|
216
|
+
]
|
217
|
+
|
218
|
+
|
219
|
+
NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS = (NESTED_TEMPLATED_STEPS_NODES, SINGLE_STEP_EDGES)
|
@@ -282,6 +282,17 @@ class Step:
|
|
282
282
|
) -> None:
|
283
283
|
"""Sets the configuration state for this ``Step``.
|
284
284
|
|
285
|
+
The so-called 'configuration state' for a given ``Step`` is backed up by
|
286
|
+
a :class:`ConfigurationState` class and is assigned to its :attr:`_configuration_state`
|
287
|
+
attribute. There are two possible ``ConfigurationStates``:
|
288
|
+
:class:`LeafConfigurationState` and :class:`NonLeafConfigurationState`.
|
289
|
+
|
290
|
+
This method sets the configuration state of this ``Step`` based on whether
|
291
|
+
or not a :attr:`config_key` is set *and exists is the ``Step's`` configuration*
|
292
|
+
(i.e. its portion of the user-suppled pipeline specification
|
293
|
+
file); any required deviation from this behavior requires special
|
294
|
+
handling.
|
295
|
+
|
285
296
|
Parameters
|
286
297
|
----------
|
287
298
|
parent_config
|
@@ -378,8 +389,9 @@ class Step:
|
|
378
389
|
) -> dict[str, list[str]]:
|
379
390
|
"""Validates a non-leaf ``Step``."""
|
380
391
|
errors = {}
|
381
|
-
|
382
|
-
|
392
|
+
nodes = self.step_graph.nodes
|
393
|
+
for node in nodes:
|
394
|
+
step = nodes[node]["step"]
|
383
395
|
if isinstance(step, IOStep):
|
384
396
|
continue
|
385
397
|
if step.name not in step_config:
|
@@ -390,7 +402,7 @@ class Step:
|
|
390
402
|
)
|
391
403
|
if step_errors:
|
392
404
|
errors.update(step_errors)
|
393
|
-
extra_steps = set(step_config.keys()) - set(
|
405
|
+
extra_steps = set(step_config.keys()) - set(nodes)
|
394
406
|
for extra_step in extra_steps:
|
395
407
|
errors[f"step {extra_step}"] = [f"{extra_step} is not a valid step."]
|
396
408
|
return errors
|
@@ -807,12 +819,42 @@ class TemplatedStep(Step, ABC):
|
|
807
819
|
The configuration for any implementations to be combined.
|
808
820
|
input_data_config
|
809
821
|
The input data configuration for the entire pipeline.
|
822
|
+
|
823
|
+
Notes
|
824
|
+
-----
|
825
|
+
A ``TemplatedStep`` is *always* assigned a :class:`NonLeafConfigurationState`
|
826
|
+
even if it has no multiplicity since (despite having no copies to make) we
|
827
|
+
still need to traverse the sub-``Steps`` to get to the one with a single
|
828
|
+
:class:`~easylink.implementation.Implementation`, i.e. the one with a
|
829
|
+
:class:`LeafConfigurationState`.
|
810
830
|
"""
|
811
|
-
|
812
|
-
self.
|
813
|
-
|
814
|
-
|
815
|
-
|
831
|
+
step_config = parent_config[self.name]
|
832
|
+
if self.config_key not in step_config:
|
833
|
+
# Special handle the step_graph update
|
834
|
+
self.step_graph = StepGraph()
|
835
|
+
self.template_step.name = self.name
|
836
|
+
self.step_graph.add_node_from_step(self.template_step)
|
837
|
+
# Special handle the slot_mappings update
|
838
|
+
input_mappings = [
|
839
|
+
InputSlotMapping(slot, self.name, slot) for slot in self.input_slots
|
840
|
+
]
|
841
|
+
output_mappings = [
|
842
|
+
OutputSlotMapping(slot, self.name, slot) for slot in self.output_slots
|
843
|
+
]
|
844
|
+
self.slot_mappings = {"input": input_mappings, "output": output_mappings}
|
845
|
+
# Add the key back to the expanded config
|
846
|
+
expanded_config = LayeredConfigTree({self.name: step_config})
|
847
|
+
else:
|
848
|
+
expanded_config = self._get_config(step_config)
|
849
|
+
num_repeats = len(expanded_config)
|
850
|
+
self.step_graph = self._update_step_graph(num_repeats)
|
851
|
+
self.slot_mappings = self._update_slot_mappings(num_repeats)
|
852
|
+
# Manually set the configuration state to non-leaf instead of relying
|
853
|
+
# on super().get_configuration_state() because that method will erroneously
|
854
|
+
# set to leaf state in the event the user didn't include the config_key
|
855
|
+
# in the pipeline specification.
|
856
|
+
self._configuration_state = NonLeafConfigurationState(
|
857
|
+
self, expanded_config, combined_implementations, input_data_config
|
816
858
|
)
|
817
859
|
|
818
860
|
def _duplicate_template_step(self) -> Step:
|
@@ -1105,9 +1147,10 @@ class ChoiceStep(Step):
|
|
1105
1147
|
initial ones are handled.
|
1106
1148
|
|
1107
1149
|
We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1150
|
+
in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
|
1151
|
+
as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
|
1152
|
+
prior to :meth:`set_configuration_state`, but the validations itself actually
|
1153
|
+
requires the updated ``StepGraph`` and ``SlotMappings``.
|
1111
1154
|
|
1112
1155
|
We do not attempt to validate the subgraph here if the 'type' key is unable
|
1113
1156
|
to be validated.
|
@@ -1136,7 +1179,7 @@ class ChoiceStep(Step):
|
|
1136
1179
|
]
|
1137
1180
|
}
|
1138
1181
|
|
1139
|
-
#
|
1182
|
+
# HACK: Update the step graph and mappings here because we need them for validation
|
1140
1183
|
self.step_graph = self._update_step_graph(subgraph)
|
1141
1184
|
self.slot_mappings = self._update_slot_mappings(subgraph)
|
1142
1185
|
# NOTE: A ChoiceStep is by definition non-leaf step
|
@@ -1163,11 +1206,11 @@ class ChoiceStep(Step):
|
|
1163
1206
|
|
1164
1207
|
Notes
|
1165
1208
|
-----
|
1166
|
-
We update the :class
|
1167
|
-
:
|
1168
|
-
:meth:`validate_step`
|
1169
|
-
:
|
1170
|
-
|
1209
|
+
We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
|
1210
|
+
in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
|
1211
|
+
as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
|
1212
|
+
prior to :meth:`set_configuration_state`, but the validations itself actually
|
1213
|
+
requires the updated ``StepGraph`` and ``SlotMappings``.
|
1171
1214
|
"""
|
1172
1215
|
|
1173
1216
|
chosen_parent_config = LayeredConfigTree(
|
@@ -1364,7 +1407,6 @@ class LeafConfigurationState(ConfigurationState):
|
|
1364
1407
|
for mapping in mappings:
|
1365
1408
|
imp_edge = mapping.remap_edge(edge)
|
1366
1409
|
implementation_edges.append(imp_edge)
|
1367
|
-
|
1368
1410
|
elif edge.target_node == self._step.name:
|
1369
1411
|
mappings = [
|
1370
1412
|
mapping
|
@@ -1520,7 +1562,6 @@ class NonLeafConfigurationState(ConfigurationState):
|
|
1520
1562
|
new_step = self._step.step_graph.nodes[mapping.child_node]["step"]
|
1521
1563
|
imp_edges = new_step.get_implementation_edges(new_edge)
|
1522
1564
|
implementation_edges.extend(imp_edges)
|
1523
|
-
|
1524
1565
|
elif edge.target_node == self._step.name:
|
1525
1566
|
mappings = [
|
1526
1567
|
mapping
|
@@ -1544,8 +1585,9 @@ class NonLeafConfigurationState(ConfigurationState):
|
|
1544
1585
|
This method recursively traverses the ``StepGraph`` and sets the configuration
|
1545
1586
|
state for each ``Step`` until reaching all leaf nodes.
|
1546
1587
|
"""
|
1547
|
-
|
1548
|
-
|
1588
|
+
nodes = self._step.step_graph.nodes
|
1589
|
+
for node in nodes:
|
1590
|
+
step = nodes[node]["step"]
|
1549
1591
|
step.set_configuration_state(
|
1550
1592
|
self.pipeline_config, self.combined_implementations, self.input_data_config
|
1551
1593
|
)
|
@@ -35,7 +35,7 @@ docs/source/api_reference/runner.rst
|
|
35
35
|
docs/source/api_reference/step.rst
|
36
36
|
docs/source/api_reference/pipeline_schema_constants/development.rst
|
37
37
|
docs/source/api_reference/pipeline_schema_constants/index.rst
|
38
|
-
docs/source/api_reference/pipeline_schema_constants/
|
38
|
+
docs/source/api_reference/pipeline_schema_constants/testing.rst
|
39
39
|
docs/source/api_reference/utilities/data_utils.rst
|
40
40
|
docs/source/api_reference/utilities/general_utils.rst
|
41
41
|
docs/source/api_reference/utilities/index.rst
|
@@ -71,7 +71,7 @@ src/easylink/images/spark_cluster/Dockerfile
|
|
71
71
|
src/easylink/images/spark_cluster/README.md
|
72
72
|
src/easylink/pipeline_schema_constants/__init__.py
|
73
73
|
src/easylink/pipeline_schema_constants/development.py
|
74
|
-
src/easylink/pipeline_schema_constants/
|
74
|
+
src/easylink/pipeline_schema_constants/testing.py
|
75
75
|
src/easylink/steps/dev/README.md
|
76
76
|
src/easylink/steps/dev/build-containers-local.sh
|
77
77
|
src/easylink/steps/dev/build-containers-remote.sh
|
@@ -112,6 +112,33 @@ tests/specifications/e2e/pipeline_expanded.yaml
|
|
112
112
|
tests/specifications/integration/environment_spark_slurm.yaml
|
113
113
|
tests/specifications/integration/pipeline.yaml
|
114
114
|
tests/specifications/integration/pipeline_spark.yaml
|
115
|
+
tests/specifications/unit/environment_minimum.yaml
|
116
|
+
tests/specifications/unit/environment_spark_slurm.yaml
|
117
|
+
tests/specifications/unit/pipeline.yaml
|
118
|
+
tests/specifications/unit/pipeline_bad_combined_implementations.yaml
|
119
|
+
tests/specifications/unit/pipeline_bad_implementation.yaml
|
120
|
+
tests/specifications/unit/pipeline_bad_loop_formatting.yaml
|
121
|
+
tests/specifications/unit/pipeline_bad_step.yaml
|
122
|
+
tests/specifications/unit/pipeline_bad_type_key.yaml
|
123
|
+
tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml
|
124
|
+
tests/specifications/unit/pipeline_combine_bad_topology.yaml
|
125
|
+
tests/specifications/unit/pipeline_combine_two_steps.yaml
|
126
|
+
tests/specifications/unit/pipeline_combine_with_extra_node.yaml
|
127
|
+
tests/specifications/unit/pipeline_combine_with_iteration.yaml
|
128
|
+
tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml
|
129
|
+
tests/specifications/unit/pipeline_combine_with_missing_node.yaml
|
130
|
+
tests/specifications/unit/pipeline_combine_with_parallel.yaml
|
131
|
+
tests/specifications/unit/pipeline_missing_implementation_name.yaml
|
132
|
+
tests/specifications/unit/pipeline_missing_implementations.yaml
|
133
|
+
tests/specifications/unit/pipeline_missing_loop_nodes.yaml
|
134
|
+
tests/specifications/unit/pipeline_missing_step.yaml
|
135
|
+
tests/specifications/unit/pipeline_missing_substeps.yaml
|
136
|
+
tests/specifications/unit/pipeline_missing_type_key.yaml
|
137
|
+
tests/specifications/unit/pipeline_nested_templated_steps.yaml
|
138
|
+
tests/specifications/unit/pipeline_out_of_order.yaml
|
139
|
+
tests/specifications/unit/pipeline_spark.yaml
|
140
|
+
tests/specifications/unit/pipeline_type_config_mismatch.yaml
|
141
|
+
tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml
|
115
142
|
tests/unit/__init__.py
|
116
143
|
tests/unit/conftest.py
|
117
144
|
tests/unit/test_cli.py
|
@@ -0,0 +1,15 @@
|
|
1
|
+
computing_environment: slurm
|
2
|
+
container_engine: singularity
|
3
|
+
slurm:
|
4
|
+
account: some-account
|
5
|
+
partition: some-partition
|
6
|
+
implementation_resources:
|
7
|
+
memory: 42
|
8
|
+
cpus: 42
|
9
|
+
time_limit: 42
|
10
|
+
spark:
|
11
|
+
workers:
|
12
|
+
num_workers: 42
|
13
|
+
cpus_per_node: 42
|
14
|
+
mem_per_node: 42
|
15
|
+
time_limit: 42
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
implementation:
|
10
|
+
name: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: simple
|
13
|
+
step_4:
|
14
|
+
implementation:
|
15
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,14 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
combined_implementation_key: foo
|
10
|
+
choice_section:
|
11
|
+
type: simple
|
12
|
+
step_4:
|
13
|
+
implementation:
|
14
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: foo # not a valid/supported implementation
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
implementation:
|
10
|
+
name: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: simple
|
13
|
+
step_4:
|
14
|
+
implementation:
|
15
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
iterate:
|
10
|
+
implementation: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: simple
|
13
|
+
step_4:
|
14
|
+
implementation:
|
15
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,18 @@
|
|
1
|
+
steps:
|
2
|
+
foo:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_1:
|
6
|
+
implementation:
|
7
|
+
name: step_1_python_pandas
|
8
|
+
step_2:
|
9
|
+
implementation:
|
10
|
+
name: step_2_python_pandas
|
11
|
+
step_3:
|
12
|
+
implementation:
|
13
|
+
name: step_3_python_pandas
|
14
|
+
choice_section:
|
15
|
+
type: simple
|
16
|
+
step_4:
|
17
|
+
implementation:
|
18
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
implementation:
|
10
|
+
name: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: foo # not a supported type
|
13
|
+
step_4:
|
14
|
+
implementation:
|
15
|
+
name: step_4_python_pandas
|
@@ -0,0 +1,18 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
combined_implementation_key: step_3_4 # incorrect key
|
9
|
+
step_3:
|
10
|
+
implementation:
|
11
|
+
name: step_3_python_pandas
|
12
|
+
choice_section:
|
13
|
+
type: simple
|
14
|
+
step_4:
|
15
|
+
combined_implementation_key: step_3_4
|
16
|
+
combined_implementations:
|
17
|
+
step_3_4:
|
18
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,18 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
iterate:
|
4
|
+
- substeps:
|
5
|
+
step_1a:
|
6
|
+
implementation:
|
7
|
+
name: step_1a_python_pandas
|
8
|
+
step_1b:
|
9
|
+
combined_implementation_key: step_1a_1b
|
10
|
+
- substeps:
|
11
|
+
step_1a:
|
12
|
+
combined_implementation_key: step_1a_1b
|
13
|
+
step_1b:
|
14
|
+
implementation:
|
15
|
+
name: step_1b_python_pandas
|
16
|
+
combined_implementations:
|
17
|
+
step_1a_1b:
|
18
|
+
name: step_1a_and_step_1b_combined_python_pandas
|
@@ -0,0 +1,16 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
combined_implementation_key: step_3_4
|
10
|
+
choice_section:
|
11
|
+
type: simple
|
12
|
+
step_4:
|
13
|
+
combined_implementation_key: step_3_4
|
14
|
+
combined_implementations:
|
15
|
+
step_3_4:
|
16
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
combined_implementation_key: step_3_4
|
7
|
+
step_3:
|
8
|
+
combined_implementation_key: step_3_4
|
9
|
+
choice_section:
|
10
|
+
type: simple
|
11
|
+
step_4:
|
12
|
+
combined_implementation_key: step_3_4
|
13
|
+
combined_implementations:
|
14
|
+
step_3_4:
|
15
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,19 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
iterate:
|
10
|
+
- implementation:
|
11
|
+
name: step_3_python_pandas
|
12
|
+
- combined_implementation_key: step_3_4
|
13
|
+
choice_section:
|
14
|
+
type: simple
|
15
|
+
step_4:
|
16
|
+
combined_implementation_key: step_3_4
|
17
|
+
combined_implementations:
|
18
|
+
step_3_4:
|
19
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,19 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
iterate:
|
10
|
+
- combined_implementation_key: step_3_4
|
11
|
+
- implementation:
|
12
|
+
name: step_3_python_pandas
|
13
|
+
choice_section:
|
14
|
+
type: simple
|
15
|
+
step_4:
|
16
|
+
combined_implementation_key: step_3_4
|
17
|
+
combined_implementations:
|
18
|
+
step_3_4:
|
19
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,17 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
name: step_1_python_pandas
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
implementation:
|
10
|
+
name: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: simple
|
13
|
+
step_4:
|
14
|
+
combined_implementation_key: step_3_4
|
15
|
+
combined_implementations:
|
16
|
+
step_3_4:
|
17
|
+
name: step_3_and_step_4_combined_python_pandas
|
@@ -0,0 +1,24 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
parallel:
|
4
|
+
- input_data_file: file1
|
5
|
+
implementation:
|
6
|
+
name: step_1_python_pandas
|
7
|
+
- input_data_file: file2
|
8
|
+
implementation:
|
9
|
+
name: step_1_python_pandas
|
10
|
+
- input_data_file: file2
|
11
|
+
combined_implementation_key: steps_1_and_2_combined
|
12
|
+
step_2:
|
13
|
+
combined_implementation_key: steps_1_and_2_combined
|
14
|
+
step_3:
|
15
|
+
implementation:
|
16
|
+
name: step_3_python_pandas
|
17
|
+
choice_section:
|
18
|
+
type: simple
|
19
|
+
step_4:
|
20
|
+
implementation:
|
21
|
+
name: step_4_python_pandas
|
22
|
+
combined_implementations:
|
23
|
+
steps_1_and_2_combined:
|
24
|
+
name: step_1_and_step_2_combined_python_pandas
|
@@ -0,0 +1,15 @@
|
|
1
|
+
steps:
|
2
|
+
step_1:
|
3
|
+
implementation:
|
4
|
+
foo: bar # missing name key
|
5
|
+
step_2:
|
6
|
+
implementation:
|
7
|
+
name: step_2_python_pandas
|
8
|
+
step_3:
|
9
|
+
implementation:
|
10
|
+
name: step_3_python_pandas
|
11
|
+
choice_section:
|
12
|
+
type: simple
|
13
|
+
step_4:
|
14
|
+
implementation:
|
15
|
+
name: step_4_python_pandas
|