easylink 0.1.4__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. {easylink-0.1.4 → easylink-0.1.6}/CHANGELOG.rst +9 -1
  2. {easylink-0.1.4 → easylink-0.1.6}/PKG-INFO +1 -1
  3. easylink-0.1.6/docs/source/api_reference/pipeline_schema_constants/testing.rst +1 -0
  4. easylink-0.1.6/src/easylink/_version.py +1 -0
  5. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/configuration.py +1 -1
  6. easylink-0.1.6/src/easylink/pipeline_schema_constants/__init__.py +12 -0
  7. easylink-0.1.4/src/easylink/pipeline_schema_constants/tests.py → easylink-0.1.6/src/easylink/pipeline_schema_constants/testing.py +90 -1
  8. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/step.py +63 -21
  9. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/PKG-INFO +1 -1
  10. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/SOURCES.txt +29 -2
  11. easylink-0.1.6/tests/specifications/unit/environment_minimum.yaml +2 -0
  12. easylink-0.1.6/tests/specifications/unit/environment_spark_slurm.yaml +15 -0
  13. easylink-0.1.6/tests/specifications/unit/pipeline.yaml +15 -0
  14. easylink-0.1.6/tests/specifications/unit/pipeline_bad_combined_implementations.yaml +14 -0
  15. easylink-0.1.6/tests/specifications/unit/pipeline_bad_implementation.yaml +15 -0
  16. easylink-0.1.6/tests/specifications/unit/pipeline_bad_loop_formatting.yaml +15 -0
  17. easylink-0.1.6/tests/specifications/unit/pipeline_bad_step.yaml +18 -0
  18. easylink-0.1.6/tests/specifications/unit/pipeline_bad_type_key.yaml +15 -0
  19. easylink-0.1.6/tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml +18 -0
  20. easylink-0.1.6/tests/specifications/unit/pipeline_combine_bad_topology.yaml +18 -0
  21. easylink-0.1.6/tests/specifications/unit/pipeline_combine_two_steps.yaml +16 -0
  22. easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_extra_node.yaml +15 -0
  23. easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_iteration.yaml +19 -0
  24. easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml +19 -0
  25. easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_missing_node.yaml +17 -0
  26. easylink-0.1.6/tests/specifications/unit/pipeline_combine_with_parallel.yaml +24 -0
  27. easylink-0.1.6/tests/specifications/unit/pipeline_missing_implementation_name.yaml +15 -0
  28. easylink-0.1.6/tests/specifications/unit/pipeline_missing_implementations.yaml +14 -0
  29. easylink-0.1.6/tests/specifications/unit/pipeline_missing_loop_nodes.yaml +14 -0
  30. easylink-0.1.6/tests/specifications/unit/pipeline_missing_step.yaml +4 -0
  31. easylink-0.1.6/tests/specifications/unit/pipeline_missing_substeps.yaml +17 -0
  32. easylink-0.1.6/tests/specifications/unit/pipeline_missing_type_key.yaml +15 -0
  33. easylink-0.1.6/tests/specifications/unit/pipeline_nested_templated_steps.yaml +21 -0
  34. easylink-0.1.6/tests/specifications/unit/pipeline_out_of_order.yaml +15 -0
  35. easylink-0.1.6/tests/specifications/unit/pipeline_spark.yaml +7 -0
  36. easylink-0.1.6/tests/specifications/unit/pipeline_type_config_mismatch.yaml +18 -0
  37. easylink-0.1.6/tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml +19 -0
  38. easylink-0.1.6/tests/unit/conftest.py +157 -0
  39. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline_graph.py +146 -14
  40. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_runner.py +5 -5
  41. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_step.py +100 -30
  42. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_validations.py +22 -9
  43. easylink-0.1.4/docs/source/api_reference/pipeline_schema_constants/tests.rst +0 -1
  44. easylink-0.1.4/src/easylink/_version.py +0 -1
  45. easylink-0.1.4/src/easylink/pipeline_schema_constants/__init__.py +0 -11
  46. easylink-0.1.4/tests/unit/conftest.py +0 -827
  47. {easylink-0.1.4 → easylink-0.1.6}/.bandit +0 -0
  48. {easylink-0.1.4 → easylink-0.1.6}/.flake8 +0 -0
  49. {easylink-0.1.4 → easylink-0.1.6}/.github/CODEOWNERS +0 -0
  50. {easylink-0.1.4 → easylink-0.1.6}/.github/pull_request_template.md +0 -0
  51. {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/build.yml +0 -0
  52. {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/deploy.yml +0 -0
  53. {easylink-0.1.4 → easylink-0.1.6}/.github/workflows/update_readme.yml +0 -0
  54. {easylink-0.1.4 → easylink-0.1.6}/.gitignore +0 -0
  55. {easylink-0.1.4 → easylink-0.1.6}/.readthedocs.yml +0 -0
  56. {easylink-0.1.4 → easylink-0.1.6}/Jenkinsfile +0 -0
  57. {easylink-0.1.4 → easylink-0.1.6}/Makefile +0 -0
  58. {easylink-0.1.4 → easylink-0.1.6}/README.rst +0 -0
  59. {easylink-0.1.4 → easylink-0.1.6}/docs/Makefile +0 -0
  60. {easylink-0.1.4 → easylink-0.1.6}/docs/nitpick-exceptions +0 -0
  61. {easylink-0.1.4 → easylink-0.1.6}/docs/source/_static/style.css +0 -0
  62. {easylink-0.1.4 → easylink-0.1.6}/docs/source/_templates/layout.html +0 -0
  63. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/configuration.rst +0 -0
  64. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/graph_components.rst +0 -0
  65. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/implementation.rst +0 -0
  66. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/index.rst +0 -0
  67. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline.rst +0 -0
  68. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_graph.rst +0 -0
  69. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema.rst +0 -0
  70. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema_constants/development.rst +0 -0
  71. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/pipeline_schema_constants/index.rst +0 -0
  72. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/rule.rst +0 -0
  73. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/runner.rst +0 -0
  74. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/step.rst +0 -0
  75. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/data_utils.rst +0 -0
  76. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/general_utils.rst +0 -0
  77. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/index.rst +0 -0
  78. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/paths.rst +0 -0
  79. {easylink-0.1.4 → easylink-0.1.6}/docs/source/api_reference/utilities/validation_utils.rst +0 -0
  80. {easylink-0.1.4 → easylink-0.1.6}/docs/source/concepts/index.rst +0 -0
  81. {easylink-0.1.4 → easylink-0.1.6}/docs/source/conf.py +0 -0
  82. {easylink-0.1.4 → easylink-0.1.6}/docs/source/glossary.rst +0 -0
  83. {easylink-0.1.4 → easylink-0.1.6}/docs/source/index.rst +0 -0
  84. {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/cli.rst +0 -0
  85. {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/index.rst +0 -0
  86. {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/tutorials/getting_started.rst +0 -0
  87. {easylink-0.1.4 → easylink-0.1.6}/docs/source/user_guide/tutorials/index.rst +0 -0
  88. {easylink-0.1.4 → easylink-0.1.6}/pyproject.toml +0 -0
  89. {easylink-0.1.4 → easylink-0.1.6}/python_versions.json +0 -0
  90. {easylink-0.1.4 → easylink-0.1.6}/pytype.cfg +0 -0
  91. {easylink-0.1.4 → easylink-0.1.6}/setup.cfg +0 -0
  92. {easylink-0.1.4 → easylink-0.1.6}/setup.py +0 -0
  93. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/__about__.py +0 -0
  94. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/__init__.py +0 -0
  95. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/cli.py +0 -0
  96. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/graph_components.py +0 -0
  97. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/images/spark_cluster/Dockerfile +0 -0
  98. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/images/spark_cluster/README.md +0 -0
  99. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/implementation.py +0 -0
  100. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/implementation_metadata.yaml +0 -0
  101. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline.py +0 -0
  102. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_graph.py +0 -0
  103. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_schema.py +0 -0
  104. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/pipeline_schema_constants/development.py +0 -0
  105. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/rule.py +0 -0
  106. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/runner.py +0 -0
  107. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/README.md +0 -0
  108. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/build-containers-local.sh +0 -0
  109. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/build-containers-remote.sh +0 -0
  110. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/create_input_files.ipynb +0 -0
  111. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_1.csv +0 -0
  112. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_1.parquet +0 -0
  113. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_2.csv +0 -0
  114. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/input_data/input_file_2.parquet +0 -0
  115. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/README.md +0 -0
  116. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/dummy_step.py +0 -0
  117. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pandas/python_pandas.def +0 -0
  118. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/README.md +0 -0
  119. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/dummy_step.py +0 -0
  120. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/python_pyspark/python_pyspark.def +0 -0
  121. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/README.md +0 -0
  122. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/dummy_step.R +0 -0
  123. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/r/r-image.def +0 -0
  124. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/steps/dev/test.py +0 -0
  125. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/__init__.py +0 -0
  126. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/data_utils.py +0 -0
  127. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/general_utils.py +0 -0
  128. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/paths.py +0 -0
  129. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/spark.smk +0 -0
  130. {easylink-0.1.4 → easylink-0.1.6}/src/easylink/utilities/validation_utils.py +0 -0
  131. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/dependency_links.txt +0 -0
  132. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/entry_points.txt +0 -0
  133. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/not-zip-safe +0 -0
  134. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/requires.txt +0 -0
  135. {easylink-0.1.4 → easylink-0.1.6}/src/easylink.egg-info/top_level.txt +0 -0
  136. {easylink-0.1.4 → easylink-0.1.6}/tests/__init__.py +0 -0
  137. {easylink-0.1.4 → easylink-0.1.6}/tests/conftest.py +0 -0
  138. {easylink-0.1.4 → easylink-0.1.6}/tests/e2e/test_easylink_run.py +0 -0
  139. {easylink-0.1.4 → easylink-0.1.6}/tests/e2e/test_step_types.py +0 -0
  140. {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake.py +0 -0
  141. {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake_slurm.py +0 -0
  142. {easylink-0.1.4 → easylink-0.1.6}/tests/integration/test_snakemake_spark.py +0 -0
  143. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/environment_local.yaml +0 -0
  144. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/input_data.yaml +0 -0
  145. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/common/pipeline.yaml +0 -0
  146. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/environment_slurm.yaml +0 -0
  147. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/pipeline.yaml +0 -0
  148. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/e2e/pipeline_expanded.yaml +0 -0
  149. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/environment_spark_slurm.yaml +0 -0
  150. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/pipeline.yaml +0 -0
  151. {easylink-0.1.4 → easylink-0.1.6}/tests/specifications/integration/pipeline_spark.yaml +0 -0
  152. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/__init__.py +0 -0
  153. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/implemented_rule_local.txt +0 -0
  154. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/implemented_rule_slurm.txt +0 -0
  155. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/pipeline_local.txt +0 -0
  156. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/pipeline_slurm.txt +0 -0
  157. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/target_rule.txt +0 -0
  158. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/rule_strings/validation_rule.txt +0 -0
  159. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_cli.py +0 -0
  160. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_config.py +0 -0
  161. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_data_utils.py +0 -0
  162. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_general_utils.py +0 -0
  163. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_graph_components.py +0 -0
  164. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_implementation.py +0 -0
  165. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline.py +0 -0
  166. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_pipeline_schema.py +0 -0
  167. {easylink-0.1.4 → easylink-0.1.6}/tests/unit/test_rule.py +0 -0
  168. {easylink-0.1.4 → easylink-0.1.6}/update_readme.py +0 -0
@@ -1,4 +1,12 @@
1
- **0.1.4 - 2/18/25**
1
+ **0.1.6 - 2/21/25**
2
+
3
+ - Move test dictionaries to yaml files
4
+
5
+ **0.1.5 - 2/20/25**
6
+
7
+ - Fix handling of templated steps when no looping or parallelism is requested
8
+
9
+ **0.1.4 - 2/20/25**
2
10
 
3
11
  - Implement duplicate_template_step method on TemplatedStep class
4
12
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: easylink
3
- Version: 0.1.4
3
+ Version: 0.1.6
4
4
  Summary: Research repository for the EasyLink ER ecosystem project.
5
5
  Home-page: https://github.com/ihmeuw/easylink
6
6
  Author: The EasyLink developers
@@ -0,0 +1 @@
1
+ .. automodule:: easylink.pipeline_schema_constants.testing
@@ -0,0 +1 @@
1
+ __version__ = "0.1.6"
@@ -97,7 +97,7 @@ class Config(LayeredConfigTree):
97
97
  def __init__(
98
98
  self,
99
99
  config_params: dict[str, Any],
100
- potential_schemas: list[PipelineSchema] | PipelineSchema = PIPELINE_SCHEMAS,
100
+ potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
101
101
  ) -> None:
102
102
  super().__init__(layers=["initial_data", "default", "user_configured"])
103
103
  self.update(DEFAULT_ENVIRONMENT, layer="default")
@@ -0,0 +1,12 @@
1
+ from easylink.pipeline_schema_constants import development, testing
2
+
3
+ ALLOWED_SCHEMA_PARAMS = {
4
+ "development": development.SCHEMA_PARAMS,
5
+ }
6
+
7
+ TESTING_SCHEMA_PARAMS = {
8
+ "integration": testing.SINGLE_STEP_SCHEMA_PARAMS,
9
+ "combine_bad_topology": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
10
+ "combine_bad_implementation_names": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
11
+ "nested_templated_steps": testing.NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS,
12
+ }
@@ -11,7 +11,14 @@ from easylink.graph_components import (
11
11
  OutputSlot,
12
12
  OutputSlotMapping,
13
13
  )
14
- from easylink.step import HierarchicalStep, InputStep, LoopStep, OutputStep, Step
14
+ from easylink.step import (
15
+ HierarchicalStep,
16
+ InputStep,
17
+ LoopStep,
18
+ OutputStep,
19
+ ParallelStep,
20
+ Step,
21
+ )
15
22
  from easylink.utilities.validation_utils import validate_input_file_dummy
16
23
 
17
24
  SINGLE_STEP_NODES = [
@@ -128,3 +135,85 @@ BAD_COMBINED_TOPOLOGY_NODES = [
128
135
  ]
129
136
 
130
137
  BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS = (BAD_COMBINED_TOPOLOGY_NODES, SINGLE_STEP_EDGES)
138
+
139
+
140
+ NESTED_TEMPLATED_STEPS_NODES = [
141
+ InputStep(),
142
+ LoopStep(
143
+ template_step=ParallelStep(
144
+ template_step=HierarchicalStep(
145
+ step_name="step_1",
146
+ input_slots=[
147
+ InputSlot(
148
+ name="step_1_main_input",
149
+ env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
150
+ validator=validate_input_file_dummy,
151
+ ),
152
+ ],
153
+ output_slots=[OutputSlot("step_1_main_output")],
154
+ nodes=[
155
+ Step(
156
+ step_name="step_1a",
157
+ input_slots=[
158
+ InputSlot(
159
+ name="step_1a_main_input",
160
+ env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
161
+ validator=validate_input_file_dummy,
162
+ ),
163
+ ],
164
+ output_slots=[OutputSlot("step_1a_main_output")],
165
+ ),
166
+ Step(
167
+ step_name="step_1b",
168
+ input_slots=[
169
+ InputSlot(
170
+ name="step_1b_main_input",
171
+ env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
172
+ validator=validate_input_file_dummy,
173
+ ),
174
+ ],
175
+ output_slots=[OutputSlot("step_1b_main_output")],
176
+ ),
177
+ ],
178
+ edges=[
179
+ EdgeParams(
180
+ source_node="step_1a",
181
+ target_node="step_1b",
182
+ output_slot="step_1a_main_output",
183
+ input_slot="step_1b_main_input",
184
+ ),
185
+ ],
186
+ input_slot_mappings=[
187
+ InputSlotMapping(
188
+ parent_slot="step_1_main_input",
189
+ child_node="step_1a",
190
+ child_slot="step_1a_main_input",
191
+ ),
192
+ ],
193
+ output_slot_mappings=[
194
+ OutputSlotMapping(
195
+ parent_slot="step_1_main_output",
196
+ child_node="step_1b",
197
+ child_slot="step_1b_main_output",
198
+ ),
199
+ ],
200
+ ),
201
+ ),
202
+ self_edges=[
203
+ EdgeParams(
204
+ source_node="step_1",
205
+ target_node="step_1",
206
+ output_slot="step_1_main_output",
207
+ input_slot="step_1_main_input",
208
+ ),
209
+ ],
210
+ ),
211
+ OutputStep(
212
+ input_slots=[
213
+ InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
214
+ ],
215
+ ),
216
+ ]
217
+
218
+
219
+ NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS = (NESTED_TEMPLATED_STEPS_NODES, SINGLE_STEP_EDGES)
@@ -282,6 +282,17 @@ class Step:
282
282
  ) -> None:
283
283
  """Sets the configuration state for this ``Step``.
284
284
 
285
+ The so-called 'configuration state' for a given ``Step`` is backed up by
286
+ a :class:`ConfigurationState` class and is assigned to its :attr:`_configuration_state`
287
+ attribute. There are two possible ``ConfigurationStates``:
288
+ :class:`LeafConfigurationState` and :class:`NonLeafConfigurationState`.
289
+
290
+ This method sets the configuration state of this ``Step`` based on whether
291
+ or not a :attr:`config_key` is set *and exists is the ``Step's`` configuration*
292
+ (i.e. its portion of the user-suppled pipeline specification
293
+ file); any required deviation from this behavior requires special
294
+ handling.
295
+
285
296
  Parameters
286
297
  ----------
287
298
  parent_config
@@ -378,8 +389,9 @@ class Step:
378
389
  ) -> dict[str, list[str]]:
379
390
  """Validates a non-leaf ``Step``."""
380
391
  errors = {}
381
- for node in self.step_graph.nodes:
382
- step = self.step_graph.nodes[node]["step"]
392
+ nodes = self.step_graph.nodes
393
+ for node in nodes:
394
+ step = nodes[node]["step"]
383
395
  if isinstance(step, IOStep):
384
396
  continue
385
397
  if step.name not in step_config:
@@ -390,7 +402,7 @@ class Step:
390
402
  )
391
403
  if step_errors:
392
404
  errors.update(step_errors)
393
- extra_steps = set(step_config.keys()) - set(self.step_graph.nodes)
405
+ extra_steps = set(step_config.keys()) - set(nodes)
394
406
  for extra_step in extra_steps:
395
407
  errors[f"step {extra_step}"] = [f"{extra_step} is not a valid step."]
396
408
  return errors
@@ -807,12 +819,42 @@ class TemplatedStep(Step, ABC):
807
819
  The configuration for any implementations to be combined.
808
820
  input_data_config
809
821
  The input data configuration for the entire pipeline.
822
+
823
+ Notes
824
+ -----
825
+ A ``TemplatedStep`` is *always* assigned a :class:`NonLeafConfigurationState`
826
+ even if it has no multiplicity since (despite having no copies to make) we
827
+ still need to traverse the sub-``Steps`` to get to the one with a single
828
+ :class:`~easylink.implementation.Implementation`, i.e. the one with a
829
+ :class:`LeafConfigurationState`.
810
830
  """
811
- num_repeats = len(self._get_config(parent_config[self.name]))
812
- self.step_graph = self._update_step_graph(num_repeats)
813
- self.slot_mappings = self._update_slot_mappings(num_repeats)
814
- super().set_configuration_state(
815
- parent_config, combined_implementations, input_data_config
831
+ step_config = parent_config[self.name]
832
+ if self.config_key not in step_config:
833
+ # Special handle the step_graph update
834
+ self.step_graph = StepGraph()
835
+ self.template_step.name = self.name
836
+ self.step_graph.add_node_from_step(self.template_step)
837
+ # Special handle the slot_mappings update
838
+ input_mappings = [
839
+ InputSlotMapping(slot, self.name, slot) for slot in self.input_slots
840
+ ]
841
+ output_mappings = [
842
+ OutputSlotMapping(slot, self.name, slot) for slot in self.output_slots
843
+ ]
844
+ self.slot_mappings = {"input": input_mappings, "output": output_mappings}
845
+ # Add the key back to the expanded config
846
+ expanded_config = LayeredConfigTree({self.name: step_config})
847
+ else:
848
+ expanded_config = self._get_config(step_config)
849
+ num_repeats = len(expanded_config)
850
+ self.step_graph = self._update_step_graph(num_repeats)
851
+ self.slot_mappings = self._update_slot_mappings(num_repeats)
852
+ # Manually set the configuration state to non-leaf instead of relying
853
+ # on super().get_configuration_state() because that method will erroneously
854
+ # set to leaf state in the event the user didn't include the config_key
855
+ # in the pipeline specification.
856
+ self._configuration_state = NonLeafConfigurationState(
857
+ self, expanded_config, combined_implementations, input_data_config
816
858
  )
817
859
 
818
860
  def _duplicate_template_step(self) -> Step:
@@ -1105,9 +1147,10 @@ class ChoiceStep(Step):
1105
1147
  initial ones are handled.
1106
1148
 
1107
1149
  We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
1108
- here as opposed to in :meth:`set_configuration_state` (as is done in :class:`TemplatedStep`)
1109
- because ``ChoiceStep`` validation happens prior to setting the configuration
1110
- state and actually requires the ``StepGraph`` and ``SlotMappings``.
1150
+ in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
1151
+ as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
1152
+ prior to :meth:`set_configuration_state`, but the validations itself actually
1153
+ requires the updated ``StepGraph`` and ``SlotMappings``.
1111
1154
 
1112
1155
  We do not attempt to validate the subgraph here if the 'type' key is unable
1113
1156
  to be validated.
@@ -1136,7 +1179,7 @@ class ChoiceStep(Step):
1136
1179
  ]
1137
1180
  }
1138
1181
 
1139
- # Handle the actual chosen step_config
1182
+ # HACK: Update the step graph and mappings here because we need them for validation
1140
1183
  self.step_graph = self._update_step_graph(subgraph)
1141
1184
  self.slot_mappings = self._update_slot_mappings(subgraph)
1142
1185
  # NOTE: A ChoiceStep is by definition non-leaf step
@@ -1163,11 +1206,11 @@ class ChoiceStep(Step):
1163
1206
 
1164
1207
  Notes
1165
1208
  -----
1166
- We update the :class:`~easylink.graph_components.StepGraph` and
1167
- :class:`SlotMappings<easylink.graph_components.SlotMapping>` in
1168
- :meth:`validate_step` as opposed to here (as is done with
1169
- :class:`TemplatedSteps<TemplatedStep>`) because ``ChoiceStep`` validation
1170
- happens prior to this but requires the ``StepGraph`` and ``SlotMappings``.
1209
+ We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
1210
+ in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
1211
+ as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
1212
+ prior to :meth:`set_configuration_state`, but the validations itself actually
1213
+ requires the updated ``StepGraph`` and ``SlotMappings``.
1171
1214
  """
1172
1215
 
1173
1216
  chosen_parent_config = LayeredConfigTree(
@@ -1364,7 +1407,6 @@ class LeafConfigurationState(ConfigurationState):
1364
1407
  for mapping in mappings:
1365
1408
  imp_edge = mapping.remap_edge(edge)
1366
1409
  implementation_edges.append(imp_edge)
1367
-
1368
1410
  elif edge.target_node == self._step.name:
1369
1411
  mappings = [
1370
1412
  mapping
@@ -1520,7 +1562,6 @@ class NonLeafConfigurationState(ConfigurationState):
1520
1562
  new_step = self._step.step_graph.nodes[mapping.child_node]["step"]
1521
1563
  imp_edges = new_step.get_implementation_edges(new_edge)
1522
1564
  implementation_edges.extend(imp_edges)
1523
-
1524
1565
  elif edge.target_node == self._step.name:
1525
1566
  mappings = [
1526
1567
  mapping
@@ -1544,8 +1585,9 @@ class NonLeafConfigurationState(ConfigurationState):
1544
1585
  This method recursively traverses the ``StepGraph`` and sets the configuration
1545
1586
  state for each ``Step`` until reaching all leaf nodes.
1546
1587
  """
1547
- for node in self._step.step_graph.nodes:
1548
- step = self._step.step_graph.nodes[node]["step"]
1588
+ nodes = self._step.step_graph.nodes
1589
+ for node in nodes:
1590
+ step = nodes[node]["step"]
1549
1591
  step.set_configuration_state(
1550
1592
  self.pipeline_config, self.combined_implementations, self.input_data_config
1551
1593
  )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: easylink
3
- Version: 0.1.4
3
+ Version: 0.1.6
4
4
  Summary: Research repository for the EasyLink ER ecosystem project.
5
5
  Home-page: https://github.com/ihmeuw/easylink
6
6
  Author: The EasyLink developers
@@ -35,7 +35,7 @@ docs/source/api_reference/runner.rst
35
35
  docs/source/api_reference/step.rst
36
36
  docs/source/api_reference/pipeline_schema_constants/development.rst
37
37
  docs/source/api_reference/pipeline_schema_constants/index.rst
38
- docs/source/api_reference/pipeline_schema_constants/tests.rst
38
+ docs/source/api_reference/pipeline_schema_constants/testing.rst
39
39
  docs/source/api_reference/utilities/data_utils.rst
40
40
  docs/source/api_reference/utilities/general_utils.rst
41
41
  docs/source/api_reference/utilities/index.rst
@@ -71,7 +71,7 @@ src/easylink/images/spark_cluster/Dockerfile
71
71
  src/easylink/images/spark_cluster/README.md
72
72
  src/easylink/pipeline_schema_constants/__init__.py
73
73
  src/easylink/pipeline_schema_constants/development.py
74
- src/easylink/pipeline_schema_constants/tests.py
74
+ src/easylink/pipeline_schema_constants/testing.py
75
75
  src/easylink/steps/dev/README.md
76
76
  src/easylink/steps/dev/build-containers-local.sh
77
77
  src/easylink/steps/dev/build-containers-remote.sh
@@ -112,6 +112,33 @@ tests/specifications/e2e/pipeline_expanded.yaml
112
112
  tests/specifications/integration/environment_spark_slurm.yaml
113
113
  tests/specifications/integration/pipeline.yaml
114
114
  tests/specifications/integration/pipeline_spark.yaml
115
+ tests/specifications/unit/environment_minimum.yaml
116
+ tests/specifications/unit/environment_spark_slurm.yaml
117
+ tests/specifications/unit/pipeline.yaml
118
+ tests/specifications/unit/pipeline_bad_combined_implementations.yaml
119
+ tests/specifications/unit/pipeline_bad_implementation.yaml
120
+ tests/specifications/unit/pipeline_bad_loop_formatting.yaml
121
+ tests/specifications/unit/pipeline_bad_step.yaml
122
+ tests/specifications/unit/pipeline_bad_type_key.yaml
123
+ tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml
124
+ tests/specifications/unit/pipeline_combine_bad_topology.yaml
125
+ tests/specifications/unit/pipeline_combine_two_steps.yaml
126
+ tests/specifications/unit/pipeline_combine_with_extra_node.yaml
127
+ tests/specifications/unit/pipeline_combine_with_iteration.yaml
128
+ tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml
129
+ tests/specifications/unit/pipeline_combine_with_missing_node.yaml
130
+ tests/specifications/unit/pipeline_combine_with_parallel.yaml
131
+ tests/specifications/unit/pipeline_missing_implementation_name.yaml
132
+ tests/specifications/unit/pipeline_missing_implementations.yaml
133
+ tests/specifications/unit/pipeline_missing_loop_nodes.yaml
134
+ tests/specifications/unit/pipeline_missing_step.yaml
135
+ tests/specifications/unit/pipeline_missing_substeps.yaml
136
+ tests/specifications/unit/pipeline_missing_type_key.yaml
137
+ tests/specifications/unit/pipeline_nested_templated_steps.yaml
138
+ tests/specifications/unit/pipeline_out_of_order.yaml
139
+ tests/specifications/unit/pipeline_spark.yaml
140
+ tests/specifications/unit/pipeline_type_config_mismatch.yaml
141
+ tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml
115
142
  tests/unit/__init__.py
116
143
  tests/unit/conftest.py
117
144
  tests/unit/test_cli.py
@@ -0,0 +1,2 @@
1
+ computing_environment: local
2
+ container_engine: undefined
@@ -0,0 +1,15 @@
1
+ computing_environment: slurm
2
+ container_engine: singularity
3
+ slurm:
4
+ account: some-account
5
+ partition: some-partition
6
+ implementation_resources:
7
+ memory: 42
8
+ cpus: 42
9
+ time_limit: 42
10
+ spark:
11
+ workers:
12
+ num_workers: 42
13
+ cpus_per_node: 42
14
+ mem_per_node: 42
15
+ time_limit: 42
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ implementation:
10
+ name: step_3_python_pandas
11
+ choice_section:
12
+ type: simple
13
+ step_4:
14
+ implementation:
15
+ name: step_4_python_pandas
@@ -0,0 +1,14 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ combined_implementation_key: foo
10
+ choice_section:
11
+ type: simple
12
+ step_4:
13
+ implementation:
14
+ name: step_4_python_pandas
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: foo # not a valid/supported implementation
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ implementation:
10
+ name: step_3_python_pandas
11
+ choice_section:
12
+ type: simple
13
+ step_4:
14
+ implementation:
15
+ name: step_4_python_pandas
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ iterate:
10
+ implementation: step_3_python_pandas
11
+ choice_section:
12
+ type: simple
13
+ step_4:
14
+ implementation:
15
+ name: step_4_python_pandas
@@ -0,0 +1,18 @@
1
+ steps:
2
+ foo:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_1:
6
+ implementation:
7
+ name: step_1_python_pandas
8
+ step_2:
9
+ implementation:
10
+ name: step_2_python_pandas
11
+ step_3:
12
+ implementation:
13
+ name: step_3_python_pandas
14
+ choice_section:
15
+ type: simple
16
+ step_4:
17
+ implementation:
18
+ name: step_4_python_pandas
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ implementation:
10
+ name: step_3_python_pandas
11
+ choice_section:
12
+ type: foo # not a supported type
13
+ step_4:
14
+ implementation:
15
+ name: step_4_python_pandas
@@ -0,0 +1,18 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ combined_implementation_key: step_3_4 # incorrect key
9
+ step_3:
10
+ implementation:
11
+ name: step_3_python_pandas
12
+ choice_section:
13
+ type: simple
14
+ step_4:
15
+ combined_implementation_key: step_3_4
16
+ combined_implementations:
17
+ step_3_4:
18
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,18 @@
1
+ steps:
2
+ step_1:
3
+ iterate:
4
+ - substeps:
5
+ step_1a:
6
+ implementation:
7
+ name: step_1a_python_pandas
8
+ step_1b:
9
+ combined_implementation_key: step_1a_1b
10
+ - substeps:
11
+ step_1a:
12
+ combined_implementation_key: step_1a_1b
13
+ step_1b:
14
+ implementation:
15
+ name: step_1b_python_pandas
16
+ combined_implementations:
17
+ step_1a_1b:
18
+ name: step_1a_and_step_1b_combined_python_pandas
@@ -0,0 +1,16 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ combined_implementation_key: step_3_4
10
+ choice_section:
11
+ type: simple
12
+ step_4:
13
+ combined_implementation_key: step_3_4
14
+ combined_implementations:
15
+ step_3_4:
16
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ combined_implementation_key: step_3_4
7
+ step_3:
8
+ combined_implementation_key: step_3_4
9
+ choice_section:
10
+ type: simple
11
+ step_4:
12
+ combined_implementation_key: step_3_4
13
+ combined_implementations:
14
+ step_3_4:
15
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,19 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ iterate:
10
+ - implementation:
11
+ name: step_3_python_pandas
12
+ - combined_implementation_key: step_3_4
13
+ choice_section:
14
+ type: simple
15
+ step_4:
16
+ combined_implementation_key: step_3_4
17
+ combined_implementations:
18
+ step_3_4:
19
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,19 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ iterate:
10
+ - combined_implementation_key: step_3_4
11
+ - implementation:
12
+ name: step_3_python_pandas
13
+ choice_section:
14
+ type: simple
15
+ step_4:
16
+ combined_implementation_key: step_3_4
17
+ combined_implementations:
18
+ step_3_4:
19
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,17 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ name: step_1_python_pandas
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ implementation:
10
+ name: step_3_python_pandas
11
+ choice_section:
12
+ type: simple
13
+ step_4:
14
+ combined_implementation_key: step_3_4
15
+ combined_implementations:
16
+ step_3_4:
17
+ name: step_3_and_step_4_combined_python_pandas
@@ -0,0 +1,24 @@
1
+ steps:
2
+ step_1:
3
+ parallel:
4
+ - input_data_file: file1
5
+ implementation:
6
+ name: step_1_python_pandas
7
+ - input_data_file: file2
8
+ implementation:
9
+ name: step_1_python_pandas
10
+ - input_data_file: file2
11
+ combined_implementation_key: steps_1_and_2_combined
12
+ step_2:
13
+ combined_implementation_key: steps_1_and_2_combined
14
+ step_3:
15
+ implementation:
16
+ name: step_3_python_pandas
17
+ choice_section:
18
+ type: simple
19
+ step_4:
20
+ implementation:
21
+ name: step_4_python_pandas
22
+ combined_implementations:
23
+ steps_1_and_2_combined:
24
+ name: step_1_and_step_2_combined_python_pandas
@@ -0,0 +1,15 @@
1
+ steps:
2
+ step_1:
3
+ implementation:
4
+ foo: bar # missing name key
5
+ step_2:
6
+ implementation:
7
+ name: step_2_python_pandas
8
+ step_3:
9
+ implementation:
10
+ name: step_3_python_pandas
11
+ choice_section:
12
+ type: simple
13
+ step_4:
14
+ implementation:
15
+ name: step_4_python_pandas