easylink 0.1.16__tar.gz → 0.1.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {easylink-0.1.16 → easylink-0.1.17}/CHANGELOG.rst +4 -0
- {easylink-0.1.16 → easylink-0.1.17}/PKG-INFO +1 -1
- easylink-0.1.17/src/easylink/_version.py +1 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/implementation.py +11 -2
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/implementation_metadata.yaml +22 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/__init__.py +1 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/testing.py +53 -1
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/rule.py +14 -1
- easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def +22 -0
- easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py +18 -0
- easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def +22 -0
- easylink-0.1.17/src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py +22 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/validation_utils.py +6 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/PKG-INFO +1 -1
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/SOURCES.txt +7 -0
- easylink-0.1.17/tests/integration/test_snakemake.py +78 -0
- easylink-0.1.17/tests/specifications/common/input_data_one_file.yaml +1 -0
- easylink-0.1.17/tests/specifications/integration/pipeline_output_dir.yaml +7 -0
- easylink-0.1.17/tests/specifications/integration/pipeline_output_dir_default.yaml +7 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/implemented_rule_local.txt +2 -2
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/implemented_rule_slurm.txt +2 -2
- easylink-0.1.17/tests/unit/test_implementation.py +92 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_rule.py +55 -7
- easylink-0.1.16/src/easylink/_version.py +0 -1
- easylink-0.1.16/tests/integration/test_snakemake.py +0 -35
- easylink-0.1.16/tests/unit/test_implementation.py +0 -26
- {easylink-0.1.16 → easylink-0.1.17}/.bandit +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.flake8 +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.github/CODEOWNERS +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.github/pull_request_template.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.github/workflows/deploy.yml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.github/workflows/update_readme.yml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.gitignore +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/.readthedocs.yml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/CONTRIBUTING.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/Jenkinsfile +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/Makefile +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/README.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/Makefile +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/nitpick-exceptions +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/_static/style.css +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/_templates/layout.html +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/cli.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/configuration.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/graph_components.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/implementation.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_graph.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/development.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/pipeline_schema_constants/testing.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/rule.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/runner.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/step.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/aggregator_utils.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/data_utils.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/general_utils.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/paths.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/splitter_utils.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/api_reference/utilities/validation_utils.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/01_step.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/02_default_implementation.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/03_slots.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/04_data_dependency.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/05_pipeline_schema.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/06_default_input.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/07_cloneable_section.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/08_cloneable_section_expanded.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/09_loopable_section.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/10_loopable_section_expanded.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/11_cloneable_section_splitter.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/12_cloneable_section_splitter_expanded.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/13_autoparallel_section.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/14_choice_section.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/15_choice_section_expanded.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/16_step_hierarchy.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/17_draws.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/18_schema_to_pipeline.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/19_schema_to_pipeline_combined.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/clustering_sub_steps.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/easylink_pipeline_schema.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/images/entity_resolution_sub_steps.drawio.png +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/concepts/pipeline_schema/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/conf.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/glossary.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/cli.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-common-pipeline.svg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-e2e-pipeline-expanded.svg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-e2e-pipeline.svg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/DAG-r-pyspark.svg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/environment_slurm.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/getting_started.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/impl-config-pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/index.rst +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_data.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_1.parquet +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_2.parquet +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/input_file_3.parquet +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/docs/source/user_guide/tutorials/r_spark_pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/pyproject.toml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/python_versions.json +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/pytype.cfg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/setup.cfg +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/setup.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/__about__.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/__init__.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/cli.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/configuration.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/devtools/implementation_creator.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/graph_components.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/images/spark_cluster/Dockerfile +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/images/spark_cluster/README.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_graph.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/pipeline_schema_constants/development.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/runner.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/step.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/README.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/build-containers-local.sh +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/build-containers-remote.sh +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/create_input_files.ipynb +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_1.csv +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_1.parquet +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_2.csv +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/input_data/input_file_2.parquet +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/README.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/dummy_step.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pandas/python_pandas.def +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/README.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/dummy_step.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/python_pyspark/python_pyspark.def +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/README.md +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/dummy_step.R +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/r/r-image.def +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/steps/dev/test.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/__init__.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/aggregator_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/data_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/general_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/paths.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/spark.smk +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink/utilities/splitter_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/dependency_links.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/entry_points.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/not-zip-safe +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/requires.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/src/easylink.egg-info/top_level.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/__init__.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/conftest.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/e2e/test_easylink_run.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/e2e/test_step_types.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_compositions.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_snakemake_slurm.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/integration/test_snakemake_spark.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/environment_local.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/input_data.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/common/pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/environment_slurm.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/e2e/pipeline_expanded.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_hierarchical_step.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_loop_step.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/embarrassingly_parallel/pipeline_parallel_step.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/environment_spark_slurm.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/integration/pipeline_spark.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/environment_minimum.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/environment_spark_slurm.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_combined_implementations.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_implementation.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_loop_formatting.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_step.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_bad_type_key.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_bad_implementation_names.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_bad_topology.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_two_steps.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_extra_node.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_iteration.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_iteration_cycle.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_missing_node.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_combine_with_parallel.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_implementation_name.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_implementations.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_loop_nodes.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_step.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_substeps.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_missing_type_key.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_nested_templated_steps.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_out_of_order.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_spark.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_type_config_mismatch.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/specifications/unit/pipeline_wrong_parallel_split_keys.yaml +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/__init__.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/conftest.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/recipe_strings/python_pandas.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/aggregation_rule.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/checkpoint_rule.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/embarrassingly_parallel_rule.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/pipeline_local.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/pipeline_slurm.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/target_rule.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/rule_strings/validation_rule.txt +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_cli.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_config.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_data_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_general_utils.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_graph_components.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_implementation_creator.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline_graph.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_pipeline_schema.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_runner.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_step.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/tests/unit/test_validations.py +0 -0
- {easylink-0.1.16 → easylink-0.1.17}/update_readme.py +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "0.1.17"
|
@@ -135,8 +135,17 @@ class Implementation:
|
|
135
135
|
|
136
136
|
@property
|
137
137
|
def outputs(self) -> dict[str, list[str]]:
|
138
|
-
"""The expected output metadata.
|
139
|
-
|
138
|
+
"""The expected output paths. If output metadata is provided, use it. Otherwise,
|
139
|
+
assume that the output is a sub-directory with the name of the output slot.
|
140
|
+
If there is only one output slot, use '.'."""
|
141
|
+
if len(self.output_slots) == 1:
|
142
|
+
return self._metadata.get("outputs", {list(self.output_slots.keys())[0]: "."})
|
143
|
+
return {
|
144
|
+
output_slot_name: self._metadata.get("outputs", {}).get(
|
145
|
+
output_slot_name, output_slot_name
|
146
|
+
)
|
147
|
+
for output_slot_name in self.output_slots
|
148
|
+
}
|
140
149
|
|
141
150
|
|
142
151
|
class NullImplementation:
|
@@ -192,3 +192,25 @@ step_1a_and_step_1b_combined_python_pandas:
|
|
192
192
|
script_cmd: python /dummy_step.py
|
193
193
|
outputs:
|
194
194
|
step_1_main_output: result.parquet
|
195
|
+
dummy_step_1_for_output_dir_example:
|
196
|
+
steps:
|
197
|
+
- step_1_for_output_dir_example
|
198
|
+
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
|
199
|
+
script_cmd: python /dummy_step_1_for_output_dir_example.py
|
200
|
+
outputs:
|
201
|
+
step_1_main_output_directory: output_dir/
|
202
|
+
dummy_step_1_for_output_dir_example_default:
|
203
|
+
steps:
|
204
|
+
- step_1_for_output_dir_example
|
205
|
+
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
|
206
|
+
script_cmd: python /dummy_step_1_for_output_dir_example.py
|
207
|
+
# leave outputs out for testing purposes
|
208
|
+
# outputs:
|
209
|
+
# step_1_main_output_directory: output_dir/
|
210
|
+
dummy_step_2_for_output_dir_example:
|
211
|
+
steps:
|
212
|
+
- step_2_for_output_dir_example
|
213
|
+
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_2_for_output_dir_example.sif
|
214
|
+
script_cmd: python /dummy_step_2_for_output_dir_example.py
|
215
|
+
outputs:
|
216
|
+
step_2_main_output: result.parquet
|
@@ -17,6 +17,7 @@ ALLOWED_SCHEMA_PARAMS = {
|
|
17
17
|
|
18
18
|
TESTING_SCHEMA_PARAMS = {
|
19
19
|
"integration": testing.SCHEMA_PARAMS_ONE_STEP,
|
20
|
+
"output_dir": testing.SCHEMA_PARAMS_OUTPUT_DIR,
|
20
21
|
"combine_bad_topology": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
|
21
22
|
"combine_bad_implementation_names": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
|
22
23
|
"nested_templated_steps": testing.SCHEMA_PARAMS_NESTED_TEMPLATED_STEPS,
|
@@ -26,7 +26,7 @@ from easylink.step import (
|
|
26
26
|
)
|
27
27
|
from easylink.utilities.aggregator_utils import concatenate_datasets
|
28
28
|
from easylink.utilities.splitter_utils import split_data_in_two
|
29
|
-
from easylink.utilities.validation_utils import validate_input_file_dummy
|
29
|
+
from easylink.utilities.validation_utils import validate_dir, validate_input_file_dummy
|
30
30
|
|
31
31
|
NODES_ONE_STEP = [
|
32
32
|
InputStep(),
|
@@ -582,3 +582,55 @@ EDGES_ONE_STEP_TWO_ISLOTS = [
|
|
582
582
|
),
|
583
583
|
]
|
584
584
|
SCHEMA_PARAMS_EP_HIERARCHICAL_STEP = (NODES_EP_HIERARCHICAL_STEP, EDGES_ONE_STEP_TWO_ISLOTS)
|
585
|
+
|
586
|
+
NODES_OUTPUT_DIR = [
|
587
|
+
InputStep(),
|
588
|
+
Step(
|
589
|
+
step_name="step_1_for_output_dir_example",
|
590
|
+
input_slots=[
|
591
|
+
InputSlot(
|
592
|
+
name="step_1_main_input",
|
593
|
+
env_var="STEP_1_MAIN_INPUT_FILE_PATHS",
|
594
|
+
validator=validate_input_file_dummy,
|
595
|
+
)
|
596
|
+
],
|
597
|
+
output_slots=[OutputSlot("step_1_main_output_directory")],
|
598
|
+
),
|
599
|
+
Step(
|
600
|
+
step_name="step_2_for_output_dir_example",
|
601
|
+
input_slots=[
|
602
|
+
InputSlot(
|
603
|
+
name="step_2_main_input",
|
604
|
+
env_var="DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH",
|
605
|
+
validator=validate_dir,
|
606
|
+
)
|
607
|
+
],
|
608
|
+
output_slots=[OutputSlot("step_2_main_output")],
|
609
|
+
),
|
610
|
+
OutputStep(
|
611
|
+
input_slots=[
|
612
|
+
InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
|
613
|
+
],
|
614
|
+
),
|
615
|
+
]
|
616
|
+
EDGES_OUTPUT_DIR = [
|
617
|
+
EdgeParams(
|
618
|
+
source_node="input_data",
|
619
|
+
target_node="step_1_for_output_dir_example",
|
620
|
+
output_slot="all",
|
621
|
+
input_slot="step_1_main_input",
|
622
|
+
),
|
623
|
+
EdgeParams(
|
624
|
+
source_node="step_1_for_output_dir_example",
|
625
|
+
target_node="step_2_for_output_dir_example",
|
626
|
+
output_slot="step_1_main_output_directory",
|
627
|
+
input_slot="step_2_main_input",
|
628
|
+
),
|
629
|
+
EdgeParams(
|
630
|
+
source_node="step_2_for_output_dir_example",
|
631
|
+
target_node="results",
|
632
|
+
output_slot="step_2_main_output",
|
633
|
+
input_slot="result",
|
634
|
+
),
|
635
|
+
]
|
636
|
+
SCHEMA_PARAMS_OUTPUT_DIR = (NODES_OUTPUT_DIR, EDGES_OUTPUT_DIR)
|
@@ -17,6 +17,7 @@ import os
|
|
17
17
|
from abc import ABC, abstractmethod
|
18
18
|
from collections.abc import Callable
|
19
19
|
from dataclasses import dataclass
|
20
|
+
from pathlib import Path
|
20
21
|
|
21
22
|
|
22
23
|
class Rule(ABC):
|
@@ -125,6 +126,18 @@ class ImplementedRule(Rule):
|
|
125
126
|
def _build_io(self) -> str:
|
126
127
|
"""Builds the input/output portion of the rule."""
|
127
128
|
log_path_chunk_adder = "-{chunk}" if self.is_embarrassingly_parallel else ""
|
129
|
+
# Handle output files vs directories
|
130
|
+
files = [path for path in self.output if Path(path).suffix != ""]
|
131
|
+
if len(files) == len(self.output):
|
132
|
+
output = self.output
|
133
|
+
elif len(files) == 0:
|
134
|
+
if len(self.output) != 1:
|
135
|
+
raise NotImplementedError("Multiple output directories is not supported.")
|
136
|
+
output = f"directory('{self.output[0]}')"
|
137
|
+
else:
|
138
|
+
raise NotImplementedError(
|
139
|
+
"Mixed output types (files and directories) is not supported."
|
140
|
+
)
|
128
141
|
io_str = (
|
129
142
|
f"""
|
130
143
|
rule:
|
@@ -132,7 +145,7 @@ rule:
|
|
132
145
|
message: "Running {self.step_name} implementation: {self.implementation_name}" """
|
133
146
|
+ self._build_input()
|
134
147
|
+ f"""
|
135
|
-
output: {
|
148
|
+
output: {output}
|
136
149
|
log: "{self.diagnostics_dir}/{self.name}-output{log_path_chunk_adder}.log"
|
137
150
|
container: "{self.image_path}" """
|
138
151
|
)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
Bootstrap: docker
|
3
|
+
From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
|
4
|
+
|
5
|
+
%files
|
6
|
+
./dummy_step_1_for_output_dir_example.py /dummy_step_1_for_output_dir_example.py
|
7
|
+
|
8
|
+
%post
|
9
|
+
# Create directories
|
10
|
+
mkdir -p /input_data
|
11
|
+
mkdir -p /extra_implementation_specific_input_data
|
12
|
+
mkdir -p /results
|
13
|
+
mkdir -p /diagnostics
|
14
|
+
|
15
|
+
# Install Python packages with specific versions
|
16
|
+
pip install pandas==2.1.2 pyarrow
|
17
|
+
|
18
|
+
%environment
|
19
|
+
export LC_ALL=C
|
20
|
+
|
21
|
+
%runscript
|
22
|
+
python /dummy_step_1_for_output_dir_example.py '$@'
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# PIPELINE_SCHEMA: output_dir
|
2
|
+
# STEP_NAME: step_1_for_output_dir_example
|
3
|
+
# REQUIREMENTS: pandas==2.1.2 pyarrow
|
4
|
+
|
5
|
+
import os
|
6
|
+
from pathlib import Path
|
7
|
+
|
8
|
+
import pandas as pd
|
9
|
+
|
10
|
+
data = pd.read_parquet(os.environ["STEP_1_MAIN_INPUT_FILE_PATHS"])
|
11
|
+
|
12
|
+
print(data)
|
13
|
+
|
14
|
+
dir_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
|
15
|
+
dir_path.mkdir(parents=True, exist_ok=True)
|
16
|
+
|
17
|
+
for i in range(3):
|
18
|
+
data.to_parquet(dir_path / f"result_{i}.parquet")
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
Bootstrap: docker
|
3
|
+
From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
|
4
|
+
|
5
|
+
%files
|
6
|
+
./dummy_step_2_for_output_dir_example.py /dummy_step_2_for_output_dir_example.py
|
7
|
+
|
8
|
+
%post
|
9
|
+
# Create directories
|
10
|
+
mkdir -p /input_data
|
11
|
+
mkdir -p /extra_implementation_specific_input_data
|
12
|
+
mkdir -p /results
|
13
|
+
mkdir -p /diagnostics
|
14
|
+
|
15
|
+
# Install Python packages with specific versions
|
16
|
+
pip install pandas==2.1.2 pyarrow
|
17
|
+
|
18
|
+
%environment
|
19
|
+
export LC_ALL=C
|
20
|
+
|
21
|
+
%runscript
|
22
|
+
python /dummy_step_2_for_output_dir_example.py '$@'
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# PIPELINE_SCHEMA: output_dir
|
2
|
+
# STEP_NAME: step_2_for_output_dir_example
|
3
|
+
# REQUIREMENTS: pandas==2.1.2 pyarrow
|
4
|
+
|
5
|
+
import os
|
6
|
+
import shutil
|
7
|
+
from pathlib import Path
|
8
|
+
|
9
|
+
import pandas as pd
|
10
|
+
|
11
|
+
dir_path = Path(os.environ["DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH"])
|
12
|
+
saved = False
|
13
|
+
|
14
|
+
for i, f in enumerate([f for f in dir_path.iterdir() if f.is_file()]):
|
15
|
+
if "snakemake" in str(f):
|
16
|
+
continue
|
17
|
+
|
18
|
+
if not saved:
|
19
|
+
shutil.copy(f, os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
|
20
|
+
saved = True
|
21
|
+
|
22
|
+
print(pd.read_parquet(f))
|
@@ -50,3 +50,9 @@ def validate_input_file_dummy(filepath: str) -> None:
|
|
50
50
|
raise LookupError(
|
51
51
|
f"Data file {filepath} is missing required column(s) {missing_columns}"
|
52
52
|
)
|
53
|
+
|
54
|
+
|
55
|
+
def validate_dir(filepath: str) -> None:
|
56
|
+
input_path = Path(filepath)
|
57
|
+
if not input_path.is_dir():
|
58
|
+
raise NotADirectoryError(f"The path {filepath} is not a directory.")
|
@@ -128,6 +128,10 @@ src/easylink/steps/dev/python_pyspark/python_pyspark.def
|
|
128
128
|
src/easylink/steps/dev/r/README.md
|
129
129
|
src/easylink/steps/dev/r/dummy_step.R
|
130
130
|
src/easylink/steps/dev/r/r-image.def
|
131
|
+
src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def
|
132
|
+
src/easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py
|
133
|
+
src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def
|
134
|
+
src/easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py
|
131
135
|
src/easylink/utilities/__init__.py
|
132
136
|
src/easylink/utilities/aggregator_utils.py
|
133
137
|
src/easylink/utilities/data_utils.py
|
@@ -146,12 +150,15 @@ tests/integration/test_snakemake_slurm.py
|
|
146
150
|
tests/integration/test_snakemake_spark.py
|
147
151
|
tests/specifications/common/environment_local.yaml
|
148
152
|
tests/specifications/common/input_data.yaml
|
153
|
+
tests/specifications/common/input_data_one_file.yaml
|
149
154
|
tests/specifications/common/pipeline.yaml
|
150
155
|
tests/specifications/e2e/environment_slurm.yaml
|
151
156
|
tests/specifications/e2e/pipeline.yaml
|
152
157
|
tests/specifications/e2e/pipeline_expanded.yaml
|
153
158
|
tests/specifications/integration/environment_spark_slurm.yaml
|
154
159
|
tests/specifications/integration/pipeline.yaml
|
160
|
+
tests/specifications/integration/pipeline_output_dir.yaml
|
161
|
+
tests/specifications/integration/pipeline_output_dir_default.yaml
|
155
162
|
tests/specifications/integration/pipeline_spark.yaml
|
156
163
|
tests/specifications/integration/embarrassingly_parallel/pipeline_hierarchical_step.yaml
|
157
164
|
tests/specifications/integration/embarrassingly_parallel/pipeline_loop_step.yaml
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# mypy: ignore-errors
|
2
|
+
import pytest
|
3
|
+
|
4
|
+
from easylink.pipeline_schema import PipelineSchema
|
5
|
+
from easylink.pipeline_schema_constants import TESTING_SCHEMA_PARAMS
|
6
|
+
from easylink.runner import main
|
7
|
+
from tests.conftest import SPECIFICATIONS_DIR
|
8
|
+
|
9
|
+
|
10
|
+
@pytest.mark.slow
|
11
|
+
def test_missing_results(test_specific_results_dir, mocker, caplog):
|
12
|
+
"""Test that the pipeline fails when a step is missing output files."""
|
13
|
+
nodes, edges = TESTING_SCHEMA_PARAMS["integration"]
|
14
|
+
mocker.patch("easylink.pipeline_schema.ALLOWED_SCHEMA_PARAMS", TESTING_SCHEMA_PARAMS)
|
15
|
+
mocker.patch(
|
16
|
+
"easylink.configuration.Config._get_schema",
|
17
|
+
return_value=PipelineSchema("integration", nodes=nodes, edges=edges),
|
18
|
+
)
|
19
|
+
|
20
|
+
## Mock implementation script call to wait 1s instead of running something
|
21
|
+
mocker.patch(
|
22
|
+
"easylink.implementation.Implementation.script_cmd",
|
23
|
+
new_callable=mocker.PropertyMock,
|
24
|
+
return_value="sleep 1s",
|
25
|
+
)
|
26
|
+
with pytest.raises(SystemExit) as exit:
|
27
|
+
main(
|
28
|
+
command="run",
|
29
|
+
pipeline_specification=SPECIFICATIONS_DIR / "integration" / "pipeline.yaml",
|
30
|
+
input_data=SPECIFICATIONS_DIR / "common/input_data.yaml",
|
31
|
+
computing_environment=SPECIFICATIONS_DIR / "common/environment_local.yaml",
|
32
|
+
results_dir=test_specific_results_dir,
|
33
|
+
)
|
34
|
+
assert exit.value.code == 1
|
35
|
+
assert "MissingOutputException" in caplog.text
|
36
|
+
|
37
|
+
|
38
|
+
@pytest.mark.slow
|
39
|
+
@pytest.mark.parametrize("outputs_specified", [True, False])
|
40
|
+
def test_outputting_a_directory(outputs_specified: bool, test_specific_results_dir, mocker):
|
41
|
+
"""Test that the pipeline fails when a step is missing output files."""
|
42
|
+
nodes, edges = TESTING_SCHEMA_PARAMS["output_dir"]
|
43
|
+
mocker.patch("easylink.pipeline_schema.ALLOWED_SCHEMA_PARAMS", TESTING_SCHEMA_PARAMS)
|
44
|
+
mocker.patch(
|
45
|
+
"easylink.configuration.Config._get_schema",
|
46
|
+
return_value=PipelineSchema("output_dir", nodes=nodes, edges=edges),
|
47
|
+
)
|
48
|
+
|
49
|
+
pipeline_yaml_name = (
|
50
|
+
"pipeline_output_dir.yaml"
|
51
|
+
if outputs_specified
|
52
|
+
else "pipeline_output_dir_default.yaml"
|
53
|
+
)
|
54
|
+
step_1_output_subdir = (
|
55
|
+
"dummy_step_1_for_output_dir_example/output_dir"
|
56
|
+
if outputs_specified
|
57
|
+
else "dummy_step_1_for_output_dir_example_default"
|
58
|
+
)
|
59
|
+
|
60
|
+
with pytest.raises(SystemExit) as exit:
|
61
|
+
main(
|
62
|
+
command="run",
|
63
|
+
pipeline_specification=SPECIFICATIONS_DIR / "integration" / pipeline_yaml_name,
|
64
|
+
input_data=SPECIFICATIONS_DIR / "common/input_data_one_file.yaml",
|
65
|
+
computing_environment=SPECIFICATIONS_DIR / "common/environment_local.yaml",
|
66
|
+
results_dir=test_specific_results_dir,
|
67
|
+
)
|
68
|
+
assert exit.value.code == 0
|
69
|
+
assert (
|
70
|
+
len(
|
71
|
+
list(
|
72
|
+
(test_specific_results_dir / "intermediate" / step_1_output_subdir).rglob(
|
73
|
+
"*.parquet"
|
74
|
+
)
|
75
|
+
)
|
76
|
+
)
|
77
|
+
== 3
|
78
|
+
)
|
@@ -0,0 +1 @@
|
|
1
|
+
input_file_1: /mnt/team/simulation_science/priv/engineering/er_ecosystem/sample_data/dummy/input_file_1.parquet
|
@@ -6,12 +6,12 @@ rule:
|
|
6
6
|
dummy_container_main_input_file_paths=['foo'],
|
7
7
|
dummy_container_secondary_input_file_paths=['bar'],
|
8
8
|
validations=['bar'],
|
9
|
-
output: ['
|
9
|
+
output: ['some/file.txt']
|
10
10
|
log: "spam/foo_rule-output.log"
|
11
11
|
container: "Multipolarity.sif"
|
12
12
|
shell:
|
13
13
|
'''
|
14
|
-
export DUMMY_CONTAINER_OUTPUT_PATHS=
|
14
|
+
export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt
|
15
15
|
export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY=spam
|
16
16
|
export DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS=foo
|
17
17
|
export DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS=bar
|
@@ -6,7 +6,7 @@ rule:
|
|
6
6
|
dummy_container_main_input_file_paths=['foo'],
|
7
7
|
dummy_container_secondary_input_file_paths=['bar'],
|
8
8
|
validations=['bar'],
|
9
|
-
output: ['
|
9
|
+
output: ['some/file.txt']
|
10
10
|
log: "spam/foo_rule-output.log"
|
11
11
|
container: "Multipolarity.sif"
|
12
12
|
resources:
|
@@ -17,7 +17,7 @@ rule:
|
|
17
17
|
slurm_extra="--output 'spam/foo_rule-slurm-%j.log'"
|
18
18
|
shell:
|
19
19
|
'''
|
20
|
-
export DUMMY_CONTAINER_OUTPUT_PATHS=
|
20
|
+
export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt
|
21
21
|
export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY=spam
|
22
22
|
export DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS=foo
|
23
23
|
export DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS=bar
|
@@ -0,0 +1,92 @@
|
|
1
|
+
# mypy: ignore-errors
|
2
|
+
import pytest
|
3
|
+
from layered_config_tree import LayeredConfigTree
|
4
|
+
from pytest_mock import MockerFixture
|
5
|
+
|
6
|
+
from easylink.graph_components import OutputSlot
|
7
|
+
from easylink.implementation import Implementation
|
8
|
+
|
9
|
+
|
10
|
+
def test__get_env_vars(mocker):
|
11
|
+
mocker.patch(
|
12
|
+
"easylink.implementation.load_yaml",
|
13
|
+
return_value={
|
14
|
+
"test": {"steps": ["this_step"], "env": {"foo": "corge", "spam": "eggs"}}
|
15
|
+
},
|
16
|
+
)
|
17
|
+
implementation = Implementation(
|
18
|
+
schema_steps=["this_step"],
|
19
|
+
implementation_config=LayeredConfigTree(
|
20
|
+
{"name": "test", "configuration": {"foo": "bar", "baz": "qux"}}
|
21
|
+
),
|
22
|
+
input_slots=[],
|
23
|
+
output_slots=[],
|
24
|
+
)
|
25
|
+
assert implementation.environment_variables == {
|
26
|
+
"foo": "bar",
|
27
|
+
"baz": "qux",
|
28
|
+
"spam": "eggs",
|
29
|
+
}
|
30
|
+
|
31
|
+
|
32
|
+
@pytest.mark.parametrize(
|
33
|
+
"output_slots, metadata_outputs, expected_outputs",
|
34
|
+
[
|
35
|
+
# one slot, defined output
|
36
|
+
(
|
37
|
+
["main_output"],
|
38
|
+
{"main_output": "main/output/filepath"},
|
39
|
+
{"main_output": "main/output/filepath"},
|
40
|
+
),
|
41
|
+
# one slot, undefined output
|
42
|
+
(["main_output"], None, {"main_output": "."}),
|
43
|
+
# two slots, defined outputs
|
44
|
+
(
|
45
|
+
["main_output", "secondary_output"],
|
46
|
+
{
|
47
|
+
"main_output": "main/output/filepath",
|
48
|
+
"secondary_output": "secondary/output/filepath",
|
49
|
+
},
|
50
|
+
{
|
51
|
+
"main_output": "main/output/filepath",
|
52
|
+
"secondary_output": "secondary/output/filepath",
|
53
|
+
},
|
54
|
+
),
|
55
|
+
# two slots, one defined and one undefined output
|
56
|
+
(
|
57
|
+
["main_output", "secondary_output"],
|
58
|
+
{"secondary_output": "secondary/output/filepath"},
|
59
|
+
{"main_output": "main_output", "secondary_output": "secondary/output/filepath"},
|
60
|
+
),
|
61
|
+
# two slots, both undefined outputs
|
62
|
+
(
|
63
|
+
["main_output", "secondary_output"],
|
64
|
+
None,
|
65
|
+
{"main_output": "main_output", "secondary_output": "secondary_output"},
|
66
|
+
),
|
67
|
+
],
|
68
|
+
)
|
69
|
+
def test_outputs(
|
70
|
+
output_slots: list[str],
|
71
|
+
metadata_outputs: dict[str, str],
|
72
|
+
expected_outputs: dict[str, str],
|
73
|
+
mocker: MockerFixture,
|
74
|
+
):
|
75
|
+
"""Test that the outputs property returns the correct output details."""
|
76
|
+
mocker.patch(
|
77
|
+
"easylink.implementation.Implementation._load_metadata",
|
78
|
+
return_value={
|
79
|
+
"steps": [
|
80
|
+
"this_step",
|
81
|
+
],
|
82
|
+
"image_path": "/path/to/image.sif",
|
83
|
+
"script_cmd": "python /path/to/script.py",
|
84
|
+
**({"outputs": metadata_outputs} if metadata_outputs else {}),
|
85
|
+
},
|
86
|
+
)
|
87
|
+
implementation = Implementation(
|
88
|
+
schema_steps=["this_step"],
|
89
|
+
implementation_config=LayeredConfigTree({"name": "test"}),
|
90
|
+
output_slots=[OutputSlot(slot) for slot in output_slots],
|
91
|
+
)
|
92
|
+
assert implementation.outputs == expected_outputs
|
@@ -49,7 +49,19 @@ def test_target_rule_build_rule():
|
|
49
49
|
|
50
50
|
|
51
51
|
@pytest.mark.parametrize("computing_environment", ["local", "slurm"])
|
52
|
-
|
52
|
+
@pytest.mark.parametrize(
|
53
|
+
"output",
|
54
|
+
[
|
55
|
+
["some/file.txt"],
|
56
|
+
["some/file.txt", "some/other/file.txt"],
|
57
|
+
["some/directory"],
|
58
|
+
["some/directory", "some/other/directory"],
|
59
|
+
["some/file.txt", "some/directory"],
|
60
|
+
],
|
61
|
+
)
|
62
|
+
def test_implemented_rule_build_rule(
|
63
|
+
computing_environment: str, output: list[str], tmp_path: Path
|
64
|
+
) -> None:
|
53
65
|
if computing_environment == "slurm":
|
54
66
|
resources = {
|
55
67
|
"slurm_partition": "'slurmpart'",
|
@@ -60,6 +72,46 @@ def test_implemented_rule_build_rule(computing_environment):
|
|
60
72
|
else:
|
61
73
|
resources = None
|
62
74
|
|
75
|
+
if output == ["some/directory", "some/other/directory"]:
|
76
|
+
pytest.xfail(
|
77
|
+
reason="NotImplementedError: Multiple output directories is not supported."
|
78
|
+
)
|
79
|
+
if output == ["some/file.txt", "some/directory"]:
|
80
|
+
pytest.xfail(
|
81
|
+
reason="NotImplementedError: Mixed output types (files and directories) is not supported."
|
82
|
+
)
|
83
|
+
|
84
|
+
# The output in the snakefile will depend on whether the output is a file or a directory.
|
85
|
+
file_path = (
|
86
|
+
Path(os.path.dirname(__file__))
|
87
|
+
/ RULE_STRINGS[f"implemented_rule_{computing_environment}"]
|
88
|
+
)
|
89
|
+
with open(file_path) as expected_file:
|
90
|
+
expected_str = expected_file.read()
|
91
|
+
|
92
|
+
if output == ["some/file.txt"]:
|
93
|
+
pass # the epxected file is already correct
|
94
|
+
elif output == ["some/file.txt", "some/other/file.txt"]:
|
95
|
+
expected_str = expected_str.replace("output: ['some/file.txt']", f"output: {output}")
|
96
|
+
expected_str = expected_str.replace(
|
97
|
+
"export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt",
|
98
|
+
"export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt,some/other/file.txt",
|
99
|
+
)
|
100
|
+
elif output == ["some/directory"]:
|
101
|
+
expected_str = expected_str.replace(
|
102
|
+
"output: ['some/file.txt']", "output: directory('some/directory')"
|
103
|
+
)
|
104
|
+
expected_str = expected_str.replace(
|
105
|
+
"export DUMMY_CONTAINER_OUTPUT_PATHS=some/file.txt",
|
106
|
+
"export DUMMY_CONTAINER_OUTPUT_PATHS=some/directory",
|
107
|
+
)
|
108
|
+
else:
|
109
|
+
raise NotImplementedError
|
110
|
+
# move the modified snakefile to tmpdir
|
111
|
+
expected_filepath = tmp_path / file_path.name
|
112
|
+
with open(expected_filepath, "w") as expected_file:
|
113
|
+
expected_file.write(expected_str)
|
114
|
+
|
63
115
|
rule = ImplementedRule(
|
64
116
|
name="foo_rule",
|
65
117
|
step_name="foo_step",
|
@@ -77,7 +129,7 @@ def test_implemented_rule_build_rule(computing_environment):
|
|
77
129
|
},
|
78
130
|
},
|
79
131
|
validations=["bar"],
|
80
|
-
output=
|
132
|
+
output=output,
|
81
133
|
resources=resources,
|
82
134
|
envvars={"eggs": "coconut"},
|
83
135
|
diagnostics_dir="spam",
|
@@ -86,11 +138,7 @@ def test_implemented_rule_build_rule(computing_environment):
|
|
86
138
|
requires_spark=False,
|
87
139
|
)
|
88
140
|
|
89
|
-
|
90
|
-
Path(os.path.dirname(__file__))
|
91
|
-
/ RULE_STRINGS[f"implemented_rule_{computing_environment}"]
|
92
|
-
)
|
93
|
-
_check_rule(rule, file_path)
|
141
|
+
_check_rule(rule, expected_filepath)
|
94
142
|
|
95
143
|
|
96
144
|
def test_embarrassingly_parallel_rule_build_rule():
|
@@ -1 +0,0 @@
|
|
1
|
-
__version__ = "0.1.16"
|