easylink 0.1.16__tar.gz → 0.1.17__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. {easylink-0.1.16 → easylink-0.1.17}/CHANGELOG.rst +4 -0
  2. {easylink-0.1.16 → easylink-0.1.17}/PKG-INFO +1 -1
  3. easylink-0.1.17/src/easylink/_version.py +1 -0
  4. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/implementation.py +11 -2
  5. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/implementation_metadata.yaml +22 -0
  6. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/__init__.py +1 -0
  7. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/testing.py +53 -1
  8. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/rule.py +14 -1
  9. easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def +22 -0
  10. easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py +18 -0
  11. easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def +22 -0
  12. easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py +22 -0
  13. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/validation_utils.py +6 -0
  14. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/PKG-INFO +1 -1
  15. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/SOURCES.txt +7 -0
  16. easylink-0.1.17/tests/integration/test_snakemake.py +78 -0
  17. easylink-0.1.17/tests/specifications/common/input_data_one_file.yaml +1 -0
  18. easylink-0.1.17/tests/specifications/integration/pipeline_output_dir.yaml +7 -0
  19. easylink-0.1.17/tests/specifications/integration/pipeline_output_dir_default.yaml +7 -0
  20. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/implemented_rule_local.txt +2 -2
  21. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/implemented_rule_slurm.txt +2 -2
  22. easylink-0.1.17/tests/unit/test_implementation.py +92 -0
  23. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_rule.py +55 -7
  24. easylink-0.1.16/src/easylink/_version.py +0 -1
  25. easylink-0.1.16/tests/integration/test_snakemake.py +0 -35
  26. easylink-0.1.16/tests/unit/test_implementation.py +0 -26
  27. {easylink-0.1.16 → easylink-0.1.17}/.bandit +0 -0
  28. {easylink-0.1.16 → easylink-0.1.17}/.flake8 +0 -0
  29. {easylink-0.1.16 → easylink-0.1.17}/.github/CODEOWNERS +0 -0
  30. {easylink-0.1.16 → easylink-0.1.17}/.github/pull_request_template.md +0 -0
  31. {easylink-0.1.16 → easylink-0.1.17}/.github/workflows/deploy.yml +0 -0
  32. {easylink-0.1.16 → easylink-0.1.17}/.github/workflows/update_readme.yml +0 -0
  33. {easylink-0.1.16 → easylink-0.1.17}/.gitignore +0 -0
  34. {easylink-0.1.16 → easylink-0.1.17}/.readthedocs.yml +0 -0
  35. {easylink-0.1.16 → easylink-0.1.17}/CONTRIBUTING.rst +0 -0
  36. {easylink-0.1.16 → easylink-0.1.17}/Jenkinsfile +0 -0
  37. {easylink-0.1.16 → easylink-0.1.17}/Makefile +0 -0
  38. {easylink-0.1.16 → easylink-0.1.17}/README.rst +0 -0
  39. {easylink-0.1.16 → easylink-0.1.17}/docs/Makefile +0 -0
  40. {easylink-0.1.16 → easylink-0.1.17}/docs/nitpick-exceptions +0 -0
  41. {easylink-0.1.16 → easylink-0.1.17}/docs/source/_static/style.css +0 -0
  42. {easylink-0.1.16 → easylink-0.1.17}/docs/source/_templates/layout.html +0 -0
  43. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/cli.rst +0 -0
  44. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/configuration.rst +0 -0
  45. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/graph_components.rst +0 -0
  46. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/implementation.rst +0 -0
  47. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/index.rst +0 -0
  48. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline.rst +0 -0
  49. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_graph.rst +0 -0
  50. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema.rst +0 -0
  51. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/development.rst +0 -0
  52. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/index.rst +0 -0
  53. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/testing.rst +0 -0
  54. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/rule.rst +0 -0
  55. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/runner.rst +0 -0
  56. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/step.rst +0 -0
  57. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/aggregator_utils.rst +0 -0
  58. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/data_utils.rst +0 -0
  59. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/general_utils.rst +0 -0
  60. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/index.rst +0 -0
  61. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/paths.rst +0 -0
  62. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/splitter_utils.rst +0 -0
  63. {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/validation_utils.rst +0 -0
  64. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/index.rst +0 -0
  65. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/01_step.drawio.png +0 -0
  66. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/02_default_implementation.drawio.png +0 -0
  67. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/03_slots.drawio.png +0 -0
  68. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/04_data_dependency.drawio.png +0 -0
  69. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/05_pipeline_schema.drawio.png +0 -0
  70. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/06_default_input.drawio.png +0 -0
  71. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/07_cloneable_section.drawio.png +0 -0
  72. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/08_cloneable_section_expanded.drawio.png +0 -0
  73. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/09_loopable_section.drawio.png +0 -0
  74. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/10_loopable_section_expanded.drawio.png +0 -0
  75. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/11_cloneable_section_splitter.drawio.png +0 -0
  76. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/12_cloneable_section_splitter_expanded.drawio.png +0 -0
  77. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/13_autoparallel_section.drawio.png +0 -0
  78. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/14_choice_section.drawio.png +0 -0
  79. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/15_choice_section_expanded.drawio.png +0 -0
  80. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/16_step_hierarchy.drawio.png +0 -0
  81. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/17_draws.drawio.png +0 -0
  82. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/18_schema_to_pipeline.drawio.png +0 -0
  83. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/19_schema_to_pipeline_combined.drawio.png +0 -0
  84. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/clustering_sub_steps.drawio.png +0 -0
  85. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/easylink_pipeline_schema.drawio.png +0 -0
  86. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/entity_resolution_sub_steps.drawio.png +0 -0
  87. {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/index.rst +0 -0
  88. {easylink-0.1.16 → easylink-0.1.17}/docs/source/conf.py +0 -0
  89. {easylink-0.1.16 → easylink-0.1.17}/docs/source/glossary.rst +0 -0
  90. {easylink-0.1.16 → easylink-0.1.17}/docs/source/index.rst +0 -0
  91. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/cli.rst +0 -0
  92. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/index.rst +0 -0
  93. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-common-pipeline.svg +0 -0
  94. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-e2e-pipeline-expanded.svg +0 -0
  95. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-e2e-pipeline.svg +0 -0
  96. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-r-pyspark.svg +0 -0
  97. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/environment_slurm.yaml +0 -0
  98. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/getting_started.rst +0 -0
  99. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/impl-config-pipeline.yaml +0 -0
  100. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/index.rst +0 -0
  101. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_data.yaml +0 -0
  102. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_1.parquet +0 -0
  103. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_2.parquet +0 -0
  104. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_3.parquet +0 -0
  105. {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/r_spark_pipeline.yaml +0 -0
  106. {easylink-0.1.16 → easylink-0.1.17}/pyproject.toml +0 -0
  107. {easylink-0.1.16 → easylink-0.1.17}/python_versions.json +0 -0
  108. {easylink-0.1.16 → easylink-0.1.17}/pytype.cfg +0 -0
  109. {easylink-0.1.16 → easylink-0.1.17}/setup.cfg +0 -0
  110. {easylink-0.1.16 → easylink-0.1.17}/setup.py +0 -0
  111. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/__about__.py +0 -0
  112. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/__init__.py +0 -0
  113. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/cli.py +0 -0
  114. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/configuration.py +0 -0
  115. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/devtools/implementation_creator.py +0 -0
  116. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/graph_components.py +0 -0
  117. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/images/spark_cluster/Dockerfile +0 -0
  118. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/images/spark_cluster/README.md +0 -0
  119. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline.py +0 -0
  120. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_graph.py +0 -0
  121. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema.py +0 -0
  122. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/development.py +0 -0
  123. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/runner.py +0 -0
  124. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/step.py +0 -0
  125. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/README.md +0 -0
  126. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/build-containers-local.sh +0 -0
  127. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/build-containers-remote.sh +0 -0
  128. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/create_input_files.ipynb +0 -0
  129. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_1.csv +0 -0
  130. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_1.parquet +0 -0
  131. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_2.csv +0 -0
  132. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_2.parquet +0 -0
  133. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/README.md +0 -0
  134. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/dummy_step.py +0 -0
  135. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/python_pandas.def +0 -0
  136. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/README.md +0 -0
  137. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/dummy_step.py +0 -0
  138. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/python_pyspark.def +0 -0
  139. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/README.md +0 -0
  140. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/dummy_step.R +0 -0
  141. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/r-image.def +0 -0
  142. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/test.py +0 -0
  143. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/__init__.py +0 -0
  144. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/aggregator_utils.py +0 -0
  145. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/data_utils.py +0 -0
  146. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/general_utils.py +0 -0
  147. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/paths.py +0 -0
  148. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/spark.smk +0 -0
  149. {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/splitter_utils.py +0 -0
  150. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/dependency_links.txt +0 -0
  151. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/entry_points.txt +0 -0
  152. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/not-zip-safe +0 -0
  153. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/requires.txt +0 -0
  154. {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/top_level.txt +0 -0
  155. {easylink-0.1.16 → easylink-0.1.17}/tests/__init__.py +0 -0
  156. {easylink-0.1.16 → easylink-0.1.17}/tests/conftest.py +0 -0
  157. {easylink-0.1.16 → easylink-0.1.17}/tests/e2e/test_easylink_run.py +0 -0
  158. {easylink-0.1.16 → easylink-0.1.17}/tests/e2e/test_step_types.py +0 -0
  159. {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_compositions.py +0 -0
  160. {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_snakemake_slurm.py +0 -0
  161. {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_snakemake_spark.py +0 -0
  162. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/environment_local.yaml +0 -0
  163. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/input_data.yaml +0 -0
  164. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/pipeline.yaml +0 -0
  165. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/environment_slurm.yaml +0 -0
  166. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/pipeline.yaml +0 -0
  167. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/pipeline_expanded.yaml +0 -0
  168. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_hierarchical_step.yaml +0 -0
  169. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_loop_step.yaml +0 -0
  170. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_parallel_step.yaml +0 -0
  171. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/environment_spark_slurm.yaml +0 -0
  172. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/pipeline.yaml +0 -0
  173. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/pipeline_spark.yaml +0 -0
  174. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/environment_minimum.yaml +0 -0
  175. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/environment_spark_slurm.yaml +0 -0
  176. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline.yaml +0 -0
  177. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_combined_implementations.yaml +0 -0
  178. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_implementation.yaml +0 -0
  179. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_loop_formatting.yaml +0 -0
  180. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_step.yaml +0 -0
  181. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_type_key.yaml +0 -0
  182. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml +0 -0
  183. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_bad_topology.yaml +0 -0
  184. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_two_steps.yaml +0 -0
  185. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_extra_node.yaml +0 -0
  186. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_iteration.yaml +0 -0
  187. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml +0 -0
  188. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_missing_node.yaml +0 -0
  189. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_parallel.yaml +0 -0
  190. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_implementation_name.yaml +0 -0
  191. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_implementations.yaml +0 -0
  192. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_loop_nodes.yaml +0 -0
  193. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_step.yaml +0 -0
  194. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_substeps.yaml +0 -0
  195. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_type_key.yaml +0 -0
  196. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_nested_templated_steps.yaml +0 -0
  197. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_out_of_order.yaml +0 -0
  198. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_spark.yaml +0 -0
  199. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_type_config_mismatch.yaml +0 -0
  200. {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml +0 -0
  201. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/__init__.py +0 -0
  202. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/conftest.py +0 -0
  203. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/recipe_strings/python_pandas.txt +0 -0
  204. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/aggregation_rule.txt +0 -0
  205. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/checkpoint_rule.txt +0 -0
  206. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/embarrassingly_parallel_rule.txt +0 -0
  207. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/pipeline_local.txt +0 -0
  208. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/pipeline_slurm.txt +0 -0
  209. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/target_rule.txt +0 -0
  210. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/validation_rule.txt +0 -0
  211. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_cli.py +0 -0
  212. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_config.py +0 -0
  213. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_data_utils.py +0 -0
  214. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_general_utils.py +0 -0
  215. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_graph_components.py +0 -0
  216. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_implementation_creator.py +0 -0
  217. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline.py +0 -0
  218. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline_graph.py +0 -0
  219. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline_schema.py +0 -0
  220. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_runner.py +0 -0
  221. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_step.py +0 -0
  222. {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_validations.py +0 -0
  223. {easylink-0.1.16 → easylink-0.1.17}/update_readme.py +0 -0
@@ -1,3 +1,7 @@
1
+ **0.1.17 - 5/14/25**
2
+
3
+ - Support directories as step intermediates
4
+
1
5
  **0.1.16 - 5/13/25**
2
6
 
3
7
  - Implement new cli command to simplify implementation creation
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: easylink
3
- Version: 0.1.16
3
+ Version: 0.1.17
4
4
  Summary: Research repository for the EasyLink ER ecosystem project.
5
5
  Home-page: https://github.com/ihmeuw/easylink
6
6
  Author: The EasyLink developers
@@ -0,0 +1 @@
1
+ __version__ = "0.1.17"
@@ -135,8 +135,17 @@ class Implementation:
135
135
 
136
136
  @property
137
137
  def outputs(self) -> dict[str, list[str]]:
138
- """The expected output metadata."""
139
- return self._metadata["outputs"]
138
+ """The expected output paths. If output metadata is provided, use it. Otherwise,
139
+ assume that the output is a sub-directory with the name of the output slot.
140
+ If there is only one output slot, use '.'."""
141
+ if len(self.output_slots) == 1:
142
+ return self._metadata.get("outputs", {list(self.output_slots.keys())[0]: "."})
143
+ return {
144
+ output_slot_name: self._metadata.get("outputs", {}).get(
145
+ output_slot_name, output_slot_name
146
+ )
147
+ for output_slot_name in self.output_slots
148
+ }
140
149
 
141
150
 
142
151
  class NullImplementation:
@@ -192,3 +192,25 @@ step_1a_and_step_1b_combined_python_pandas:
192
192
  script_cmd: python /dummy_step.py
193
193
  outputs:
194
194
  step_1_main_output: result.parquet
195
+ dummy_step_1_for_output_dir_example:
196
+ steps:
197
+ - step_1_for_output_dir_example
198
+ image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
199
+ script_cmd: python /dummy_step_1_for_output_dir_example.py
200
+ outputs:
201
+ step_1_main_output_directory: output_dir/
202
+ dummy_step_1_for_output_dir_example_default:
203
+ steps:
204
+ - step_1_for_output_dir_example
205
+ image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
206
+ script_cmd: python /dummy_step_1_for_output_dir_example.py
207
+ # leave outputs out for testing purposes
208
+ # outputs:
209
+ # step_1_main_output_directory: output_dir/
210
+ dummy_step_2_for_output_dir_example:
211
+ steps:
212
+ - step_2_for_output_dir_example
213
+ image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_2_for_output_dir_example.sif
214
+ script_cmd: python /dummy_step_2_for_output_dir_example.py
215
+ outputs:
216
+ step_2_main_output: result.parquet
@@ -17,6 +17,7 @@ ALLOWED_SCHEMA_PARAMS = {
17
17
 
18
18
  TESTING_SCHEMA_PARAMS = {
19
19
  "integration": testing.SCHEMA_PARAMS_ONE_STEP,
20
+ "output_dir": testing.SCHEMA_PARAMS_OUTPUT_DIR,
20
21
  "combine_bad_topology": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
21
22
  "combine_bad_implementation_names": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
22
23
  "nested_templated_steps": testing.SCHEMA_PARAMS_NESTED_TEMPLATED_STEPS,
@@ -26,7 +26,7 @@ from easylink.step import (
26
26
  )
27
27
  from easylink.utilities.aggregator_utils import concatenate_datasets
28
28
  from easylink.utilities.splitter_utils import split_data_in_two
29
- from easylink.utilities.validation_utils import validate_input_file_dummy
29
+ from easylink.utilities.validation_utils import validate_dir, validate_input_file_dummy
30
30
 
31
31
  NODES_ONE_STEP = [
32
32
  InputStep(),
@@ -582,3 +582,55 @@ EDGES_ONE_STEP_TWO_ISLOTS = [
582
582
  ),
583
583
  ]
584
584
  SCHEMA_PARAMS_EP_HIERARCHICAL_STEP = (NODES_EP_HIERARCHICAL_STEP, EDGES_ONE_STEP_TWO_ISLOTS)
585
+
586
+ NODES_OUTPUT_DIR = [
587
+ InputStep(),
588
+ Step(
589
+ step_name="step_1_for_output_dir_example",
590
+ input_slots=[
591
+ InputSlot(
592
+ name="step_1_main_input",
593
+ env_var="STEP_1_MAIN_INPUT_FILE_PATHS",
594
+ validator=validate_input_file_dummy,
595
+ )
596
+ ],
597
+ output_slots=[OutputSlot("step_1_main_output_directory")],
598
+ ),
599
+ Step(
600
+ step_name="step_2_for_output_dir_example",
601
+ input_slots=[
602
+ InputSlot(
603
+ name="step_2_main_input",
604
+ env_var="DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH",
605
+ validator=validate_dir,
606
+ )
607
+ ],
608
+ output_slots=[OutputSlot("step_2_main_output")],
609
+ ),
610
+ OutputStep(
611
+ input_slots=[
612
+ InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
613
+ ],
614
+ ),
615
+ ]
616
+ EDGES_OUTPUT_DIR = [
617
+ EdgeParams(
618
+ source_node="input_data",
619
+ target_node="step_1_for_output_dir_example",
620
+ output_slot="all",
621
+ input_slot="step_1_main_input",
622
+ ),
623
+ EdgeParams(
624
+ source_node="step_1_for_output_dir_example",
625
+ target_node="step_2_for_output_dir_example",
626
+ output_slot="step_1_main_output_directory",
627
+ input_slot="step_2_main_input",
628
+ ),
629
+ EdgeParams(
630
+ source_node="step_2_for_output_dir_example",
631
+ target_node="results",
632
+ output_slot="step_2_main_output",
633
+ input_slot="result",
634
+ ),
635
+ ]
636
+ SCHEMA_PARAMS_OUTPUT_DIR = (NODES_OUTPUT_DIR, EDGES_OUTPUT_DIR)
@@ -17,6 +17,7 @@ import os
17
17
  from abc import ABC, abstractmethod
18
18
  from collections.abc import Callable
19
19
  from dataclasses import dataclass
20
+ from pathlib import Path
20
21
 
21
22
 
22
23
  class Rule(ABC):
@@ -125,6 +126,18 @@ class ImplementedRule(Rule):
125
126
  def _build_io(self) -> str:
126
127
  """Builds the input/output portion of the rule."""
127
128
  log_path_chunk_adder = "-{chunk}" if self.is_embarrassingly_parallel else ""
129
+ # Handle output files vs directories
130
+ files = [path for path in self.output if Path(path).suffix != ""]
131
+ if len(files) == len(self.output):
132
+ output = self.output
133
+ elif len(files) == 0:
134
+ if len(self.output) != 1:
135
+ raise NotImplementedError("Multiple output directories is not supported.")
136
+ output = f"directory('{self.output[0]}')"
137
+ else:
138
+ raise NotImplementedError(
139
+ "Mixed output types (files and directories) is not supported."
140
+ )
128
141
  io_str = (
129
142
  f"""
130
143
  rule:
@@ -132,7 +145,7 @@ rule:
132
145
  message: "Running {self.step_name} implementation: {self.implementation_name}" """
133
146
  + self._build_input()
134
147
  + f"""
135
- output: {self.output}
148
+ output: {output}
136
149
  log: "{self.diagnostics_dir}/{self.name}-output{log_path_chunk_adder}.log"
137
150
  container: "{self.image_path}" """
138
151
  )
@@ -0,0 +1,22 @@
1
+
2
+ Bootstrap: docker
3
+ From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
4
+
5
+ %files
6
+ ./dummy_step_1_for_output_dir_example.py /dummy_step_1_for_output_dir_example.py
7
+
8
+ %post
9
+ # Create directories
10
+ mkdir -p /input_data
11
+ mkdir -p /extra_implementation_specific_input_data
12
+ mkdir -p /results
13
+ mkdir -p /diagnostics
14
+
15
+ # Install Python packages with specific versions
16
+ pip install pandas==2.1.2 pyarrow
17
+
18
+ %environment
19
+ export LC_ALL=C
20
+
21
+ %runscript
22
+ python /dummy_step_1_for_output_dir_example.py '$@'
@@ -0,0 +1,18 @@
1
+ # PIPELINE_SCHEMA: output_dir
2
+ # STEP_NAME: step_1_for_output_dir_example
3
+ # REQUIREMENTS: pandas==2.1.2 pyarrow
4
+
5
+ import os
6
+ from pathlib import Path
7
+
8
+ import pandas as pd
9
+
10
+ data = pd.read_parquet(os.environ["STEP_1_MAIN_INPUT_FILE_PATHS"])
11
+
12
+ print(data)
13
+
14
+ dir_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
15
+ dir_path.mkdir(parents=True, exist_ok=True)
16
+
17
+ for i in range(3):
18
+ data.to_parquet(dir_path / f"result_{i}.parquet")
@@ -0,0 +1,22 @@
1
+
2
+ Bootstrap: docker
3
+ From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
4
+
5
+ %files
6
+ ./dummy_step_2_for_output_dir_example.py /dummy_step_2_for_output_dir_example.py
7
+
8
+ %post
9
+ # Create directories
10
+ mkdir -p /input_data
11
+ mkdir -p /extra_implementation_specific_input_data
12
+ mkdir -p /results
13
+ mkdir -p /diagnostics
14
+
15
+ # Install Python packages with specific versions
16
+ pip install pandas==2.1.2 pyarrow
17
+
18
+ %environment
19
+ export LC_ALL=C
20
+
21
+ %runscript
22
+ python /dummy_step_2_for_output_dir_example.py '$@'
@@ -0,0 +1,22 @@
1
+ # PIPELINE_SCHEMA: output_dir
2
+ # STEP_NAME: step_2_for_output_dir_example
3
+ # REQUIREMENTS: pandas==2.1.2 pyarrow
4
+
5
+ import os
6
+ import shutil
7
+ from pathlib import Path
8
+
9
+ import pandas as pd
10
+
11
+ dir_path = Path(os.environ["DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH"])
12
+ saved = False
13
+
14
+ for i, f in enumerate([f for f in dir_path.iterdir() if f.is_file()]):
15
+ if "snakemake" in str(f):
16
+ continue
17
+
18
+ if not saved:
19
+ shutil.copy(f, os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
20
+ saved = True
21
+
22
+ print(pd.read_parquet(f))
@@ -50,3 +50,9 @@ def validate_input_file_dummy(filepath: str) -> None:
50
50
  raise LookupError(
51
51
  f"Data file {filepath} is missing required column(s) {missing_columns}"
52
52
  )
53
+
54
+
55
+ def validate_dir(filepath: str) -> None:
56
+ input_path = Path(filepath)
57
+ if not input_path.is_dir():
58
+ raise NotADirectoryError(f"The path {filepath} is not a directory.")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: easylink
3
- Version: 0.1.16
3
+ Version: 0.1.17
4
4
  Summary: Research repository for the EasyLink ER ecosystem project.
5
5
  Home-page: https://github.com/ihmeuw/easylink
6
6
  Author: The EasyLink developers
@@ -128,6 +128,10 @@ src/easylink/steps/dev/python_pyspark/python_pyspark.def
128
128
  src/easylink/steps/dev/r/README.md
129
129
  src/easylink/steps/dev/r/dummy_step.R
130
130
  src/easylink/steps/dev/r/r-image.def
131
+ src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def
132
+ src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py
133
+ src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def
134
+ src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py
131
135
  src/easylink/utilities/__init__.py
132
136
  src/easylink/utilities/aggregator_utils.py
133
137
  src/easylink/utilities/data_utils.py
@@ -146,12 +150,15 @@ tests/integration/test_snakemake_slurm.py
146
150
  tests/integration/test_snakemake_spark.py
147
151
  tests/specifications/common/environment_local.yaml
148
152
  tests/specifications/common/input_data.yaml
153
+ tests/specifications/common/input_data_one_file.yaml
149
154
  tests/specifications/common/pipeline.yaml
150
155
  tests/specifications/e2e/environment_slurm.yaml
151
156
  tests/specifications/e2e/pipeline.yaml
152
157
  tests/specifications/e2e/pipeline_expanded.yaml
153
158
  tests/specifications/integration/environment_spark_slurm.yaml
154
159
  tests/specifications/integration/pipeline.yaml
160
+ tests/specifications/integration/pipeline_output_dir.yaml
161
+ tests/specifications/integration/pipeline_output_dir_default.yaml
155
162
  tests/specifications/integration/pipeline_spark.yaml
156
163
  tests/specifications/integration/embarrassingly_parallel/pipeline_hierarchical_step.yaml
157
164
  tests/specifications/integration/embarrassingly_parallel/pipeline_loop_step.yaml
@@ -0,0 +1,78 @@
1
+ # mypy: ignore-errors
2
+ import pytest
3
+
4
+ from easylink.pipeline_schema import PipelineSchema
5
+ from easylink.pipeline_schema_constants import TESTING_SCHEMA_PARAMS
6
+ from easylink.runner import main
7
+ from tests.conftest import SPECIFICATIONS_DIR
8
+
9
+
10
+ @pytest.mark.slow
11
+ def test_missing_results(test_specific_results_dir, mocker, caplog):
12
+ """Test that the pipeline fails when a step is missing output files."""
13
+ nodes, edges = TESTING_SCHEMA_PARAMS["integration"]
14
+ mocker.patch("easylink.pipeline_schema.ALLOWED_SCHEMA_PARAMS", TESTING_SCHEMA_PARAMS)
15
+ mocker.patch(
16
+ "easylink.configuration.Config._get_schema",
17
+ return_value=PipelineSchema("integration", nodes=nodes, edges=edges),
18
+ )
19
+
20
+ ## Mock implementation script call to wait 1s instead of running something
21
+ mocker.patch(
22
+ "easylink.implementation.Implementation.script_cmd",
23
+ new_callable=mocker.PropertyMock,
24
+ return_value="sleep 1s",
25
+ )
26
+ with pytest.raises(SystemExit) as exit:
27
+ main(
28
+ command="run",
29
+ pipeline_specification=SPECIFICATIONS_DIR / "integration" / "pipeline.yaml",
30
+ input_data=SPECIFICATIONS_DIR / "common/input_data.yaml",
31
+ computing_environment=SPECIFICATIONS_DIR / "common/environment_local.yaml",
32
+ results_dir=test_specific_results_dir,
33
+ )
34
+ assert exit.value.code == 1
35
+ assert "MissingOutputException" in caplog.text
36
+
37
+
38
+ @pytest.mark.slow
39
+ @pytest.mark.parametrize("outputs_specified", [True, False])
40
+ def test_outputting_a_directory(outputs_specified: bool, test_specific_results_dir, mocker):
41
+ """Test that the pipeline fails when a step is missing output files."""
42
+ nodes, edges = TESTING_SCHEMA_PARAMS["output_dir"]
43
+ mocker.patch("easylink.pipeline_schema.ALLOWED_SCHEMA_PARAMS", TESTING_SCHEMA_PARAMS)
44
+ mocker.patch(
45
+ "easylink.configuration.Config._get_schema",
46
+ return_value=PipelineSchema("output_dir", nodes=nodes, edges=edges),
47
+ )
48
+
49
+ pipeline_yaml_name = (
50
+ "pipeline_output_dir.yaml"
51
+ if outputs_specified
52
+ else "pipeline_output_dir_default.yaml"
53
+ )
54
+ step_1_output_subdir = (
55
+ "dummy_step_1_for_output_dir_example/output_dir"
56
+ if outputs_specified
57
+ else "dummy_step_1_for_output_dir_example_default"
58
+ )
59
+
60
+ with pytest.raises(SystemExit) as exit:
61
+ main(
62
+ command="run",
63
+ pipeline_specification=SPECIFICATIONS_DIR / "integration" / pipeline_yaml_name,
64
+ input_data=SPECIFICATIONS_DIR / "common/input_data_one_file.yaml",
65
+ computing_environment=SPECIFICATIONS_DIR / "common/environment_local.yaml",
66
+ results_dir=test_specific_results_dir,
67
+ )
68
+ assert exit.value.code == 0
69
+ assert (
70
+ len(
71
+ list(
72
+ (test_specific_results_dir / "intermediate" / step_1_output_subdir).rglob(
73
+ "*.parquet"
74
+ )
75
+ )
76
+ )
77
+ == 3
78
+ )
@@ -0,0 +1 @@
1
+ input_file_1: /mnt/team/simulation_science/priv/engineering/er_ecosystem/sample_data/dummy/input_file_1.parquet
@@ -0,0 +1,7 @@
1
+ steps:
2
+ step_1_for_output_dir_example:
3
+ implementation:
4
+ name: dummy_step_1_for_output_dir_example
5
+ step_2_for_output_dir_example:
6
+ implementation:
7
+ name: dummy_step_2_for_output_dir_example
@@ -0,0 +1,7 @@
1
+ steps:
2
+ step_1_for_output_dir_example:
3
+ implementation:
4
+ name: dummy_step_1_for_output_dir_example_default
5
+ step_2_for_output_dir_example:
6
+ implementation:
7
+ name: dummy_step_2_for_output_dir_example
@@ -6,12 +6,12 @@ rule:
6
6
  dummy_container_main_input_file_paths=['foo'],
7
7
  dummy_container_secondary_input_file_paths=['bar'],
8
8
  validations=['bar'],
9
- output: ['baz']
9
+ output: ['some/file.txt']
10
10
  log: "spam/foo_rule-output.log"
11
11
  container: "Multipolarity.sif"
12
12
  shell:
13
13
  '''
14
- export DUMMY_CONTAINER_OUTPUT_PATHS=baz
14
+ export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt
15
15
  export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY=spam
16
16
  export DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS=foo
17
17
  export DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS=bar
@@ -6,7 +6,7 @@ rule:
6
6
  dummy_container_main_input_file_paths=['foo'],
7
7
  dummy_container_secondary_input_file_paths=['bar'],
8
8
  validations=['bar'],
9
- output: ['baz']
9
+ output: ['some/file.txt']
10
10
  log: "spam/foo_rule-output.log"
11
11
  container: "Multipolarity.sif"
12
12
  resources:
@@ -17,7 +17,7 @@ rule:
17
17
  slurm_extra="--output 'spam/foo_rule-slurm-%j.log'"
18
18
  shell:
19
19
  '''
20
- export DUMMY_CONTAINER_OUTPUT_PATHS=baz
20
+ export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt
21
21
  export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY=spam
22
22
  export DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS=foo
23
23
  export DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS=bar
@@ -0,0 +1,92 @@
1
+ # mypy: ignore-errors
2
+ import pytest
3
+ from layered_config_tree import LayeredConfigTree
4
+ from pytest_mock import MockerFixture
5
+
6
+ from easylink.graph_components import OutputSlot
7
+ from easylink.implementation import Implementation
8
+
9
+
10
+ def test__get_env_vars(mocker):
11
+ mocker.patch(
12
+ "easylink.implementation.load_yaml",
13
+ return_value={
14
+ "test": {"steps": ["this_step"], "env": {"foo": "corge", "spam": "eggs"}}
15
+ },
16
+ )
17
+ implementation = Implementation(
18
+ schema_steps=["this_step"],
19
+ implementation_config=LayeredConfigTree(
20
+ {"name": "test", "configuration": {"foo": "bar", "baz": "qux"}}
21
+ ),
22
+ input_slots=[],
23
+ output_slots=[],
24
+ )
25
+ assert implementation.environment_variables == {
26
+ "foo": "bar",
27
+ "baz": "qux",
28
+ "spam": "eggs",
29
+ }
30
+
31
+
32
+ @pytest.mark.parametrize(
33
+ "output_slots, metadata_outputs, expected_outputs",
34
+ [
35
+ # one slot, defined output
36
+ (
37
+ ["main_output"],
38
+ {"main_output": "main/output/filepath"},
39
+ {"main_output": "main/output/filepath"},
40
+ ),
41
+ # one slot, undefined output
42
+ (["main_output"], None, {"main_output": "."}),
43
+ # two slots, defined outputs
44
+ (
45
+ ["main_output", "secondary_output"],
46
+ {
47
+ "main_output": "main/output/filepath",
48
+ "secondary_output": "secondary/output/filepath",
49
+ },
50
+ {
51
+ "main_output": "main/output/filepath",
52
+ "secondary_output": "secondary/output/filepath",
53
+ },
54
+ ),
55
+ # two slots, one defined and one undefined output
56
+ (
57
+ ["main_output", "secondary_output"],
58
+ {"secondary_output": "secondary/output/filepath"},
59
+ {"main_output": "main_output", "secondary_output": "secondary/output/filepath"},
60
+ ),
61
+ # two slots, both undefined outputs
62
+ (
63
+ ["main_output", "secondary_output"],
64
+ None,
65
+ {"main_output": "main_output", "secondary_output": "secondary_output"},
66
+ ),
67
+ ],
68
+ )
69
+ def test_outputs(
70
+ output_slots: list[str],
71
+ metadata_outputs: dict[str, str],
72
+ expected_outputs: dict[str, str],
73
+ mocker: MockerFixture,
74
+ ):
75
+ """Test that the outputs property returns the correct output details."""
76
+ mocker.patch(
77
+ "easylink.implementation.Implementation._load_metadata",
78
+ return_value={
79
+ "steps": [
80
+ "this_step",
81
+ ],
82
+ "image_path": "/path/to/image.sif",
83
+ "script_cmd": "python /path/to/script.py",
84
+ **({"outputs": metadata_outputs} if metadata_outputs else {}),
85
+ },
86
+ )
87
+ implementation = Implementation(
88
+ schema_steps=["this_step"],
89
+ implementation_config=LayeredConfigTree({"name": "test"}),
90
+ output_slots=[OutputSlot(slot) for slot in output_slots],
91
+ )
92
+ assert implementation.outputs == expected_outputs
@@ -49,7 +49,19 @@ def test_target_rule_build_rule():
49
49
 
50
50
 
51
51
  @pytest.mark.parametrize("computing_environment", ["local", "slurm"])
52
- def test_implemented_rule_build_rule(computing_environment):
52
+ @pytest.mark.parametrize(
53
+ "output",
54
+ [
55
+ ["some/file.txt"],
56
+ ["some/file.txt", "some/other/file.txt"],
57
+ ["some/directory"],
58
+ ["some/directory", "some/other/directory"],
59
+ ["some/file.txt", "some/directory"],
60
+ ],
61
+ )
62
+ def test_implemented_rule_build_rule(
63
+ computing_environment: str, output: list[str], tmp_path: Path
64
+ ) -> None:
53
65
  if computing_environment == "slurm":
54
66
  resources = {
55
67
  "slurm_partition": "'slurmpart'",
@@ -60,6 +72,46 @@ def test_implemented_rule_build_rule(computing_environment):
60
72
  else:
61
73
  resources = None
62
74
 
75
+ if output == ["some/directory", "some/other/directory"]:
76
+ pytest.xfail(
77
+ reason="NotImplementedError: Multiple output directories is not supported."
78
+ )
79
+ if output == ["some/file.txt", "some/directory"]:
80
+ pytest.xfail(
81
+ reason="NotImplementedError: Mixed output types (files and directories) is not supported."
82
+ )
83
+
84
+ # The output in the snakefile will depend on whether the output is a file or a directory.
85
+ file_path = (
86
+ Path(os.path.dirname(__file__))
87
+ / RULE_STRINGS[f"implemented_rule_{computing_environment}"]
88
+ )
89
+ with open(file_path) as expected_file:
90
+ expected_str = expected_file.read()
91
+
92
+ if output == ["some/file.txt"]:
93
+ pass # the epxected file is already correct
94
+ elif output == ["some/file.txt", "some/other/file.txt"]:
95
+ expected_str = expected_str.replace("output: ['some/file.txt']", f"output: {output}")
96
+ expected_str = expected_str.replace(
97
+ "export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt",
98
+ "export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt,some/other/file.txt",
99
+ )
100
+ elif output == ["some/directory"]:
101
+ expected_str = expected_str.replace(
102
+ "output: ['some/file.txt']", "output: directory('some/directory')"
103
+ )
104
+ expected_str = expected_str.replace(
105
+ "export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt",
106
+ "export DUMMY_CONTAINER_OUTPUT_PATHS=some/directory",
107
+ )
108
+ else:
109
+ raise NotImplementedError
110
+ # move the modified snakefile to tmpdir
111
+ expected_filepath = tmp_path / file_path.name
112
+ with open(expected_filepath, "w") as expected_file:
113
+ expected_file.write(expected_str)
114
+
63
115
  rule = ImplementedRule(
64
116
  name="foo_rule",
65
117
  step_name="foo_step",
@@ -77,7 +129,7 @@ def test_implemented_rule_build_rule(computing_environment):
77
129
  },
78
130
  },
79
131
  validations=["bar"],
80
- output=["baz"],
132
+ output=output,
81
133
  resources=resources,
82
134
  envvars={"eggs": "coconut"},
83
135
  diagnostics_dir="spam",
@@ -86,11 +138,7 @@ def test_implemented_rule_build_rule(computing_environment):
86
138
  requires_spark=False,
87
139
  )
88
140
 
89
- file_path = (
90
- Path(os.path.dirname(__file__))
91
- / RULE_STRINGS[f"implemented_rule_{computing_environment}"]
92
- )
93
- _check_rule(rule, file_path)
141
+ _check_rule(rule, expected_filepath)
94
142
 
95
143
 
96
144
  def test_embarrassingly_parallel_rule_build_rule():
@@ -1 +0,0 @@
1
- __version__ = "0.1.16"