easylink 0.1.18__py3-none-any.whl → 0.1.19__py3-none-any.whl

easylink/implementation_metadata.yaml

@@ -1,14 +1,18 @@
 step_1_python_pandas:
   steps:
   - step_1
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
 step_1a_python_pandas:
   steps:
   - step_1a
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -17,7 +21,9 @@ step_1a_python_pandas:
 step_1b_python_pandas:
   steps:
   - step_1b
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -26,21 +32,27 @@ step_1b_python_pandas:
 step_2_python_pandas:
   steps:
   - step_2
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
 step_3_python_pandas:
   steps:
   - step_3
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_3_main_output: result.parquet
 step_4_python_pandas:
   steps:
   - step_4
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -49,7 +61,9 @@ step_4_python_pandas:
 step_5_python_pandas:
   steps:
   - step_5
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -58,7 +72,9 @@ step_5_python_pandas:
 step_6_python_pandas:
   steps:
   - step_6
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -67,7 +83,9 @@ step_6_python_pandas:
 step_4a_python_pandas:
   steps:
   - step_4a
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -76,7 +94,9 @@ step_4a_python_pandas:
 step_4b_python_pandas:
   steps:
   - step_4b
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -85,7 +105,9 @@ step_4b_python_pandas:
 step_4b_r:
   steps:
   - step_4b
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
+  image_name: r-image.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 9410af1317aabc332604cbec33b59d42
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -94,7 +116,9 @@ step_4b_r:
 step_1_python_pyspark:
   steps:
   - step_1
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
+  image_name: python_pyspark.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -102,7 +126,9 @@ step_1_python_pyspark:
 step_2_python_pyspark:
   steps:
   - step_2
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
+  image_name: python_pyspark.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -110,7 +136,9 @@ step_2_python_pyspark:
 step_3_python_pyspark:
   steps:
   - step_3
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
+  image_name: python_pyspark.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_3_main_output: result.parquet
@@ -118,7 +146,9 @@ step_3_python_pyspark:
 step_4_python_pyspark:
   steps:
   - step_4
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
+  image_name: python_pyspark.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
   script_cmd: python3 /code/dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -127,7 +157,9 @@ step_4_python_pyspark:
 step_1_r:
   steps:
   - step_1
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
+  image_name: r-image.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 9410af1317aabc332604cbec33b59d42
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_1_main_output: result.parquet
@@ -135,7 +167,9 @@ step_1_r:
 step_2_r:
   steps:
   - step_2
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
+  image_name: r-image.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 9410af1317aabc332604cbec33b59d42
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_2_main_output: result.parquet
@@ -143,7 +177,9 @@ step_2_r:
 step_3_r:
   steps:
   - step_3
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
+  image_name: r-image.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 9410af1317aabc332604cbec33b59d42
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_3_main_output: result.parquet
@@ -151,7 +187,9 @@ step_3_r:
 step_4_r:
   steps:
   - step_4
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
+  image_name: r-image.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 9410af1317aabc332604cbec33b59d42
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -162,7 +200,9 @@ step_1_and_step_2_combined_python_pandas:
   steps:
   - step_1
   - step_2
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -170,7 +210,9 @@ step_1_and_step_2_parallel_python_pandas:
   steps:
   - step_1
   - step_2
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: STEP_1_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,STEP_2_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS
@@ -180,7 +222,9 @@ step_3_and_step_4_combined_python_pandas:
   steps:
   - step_3
   - step_4
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_4_main_output: result.parquet
@@ -188,26 +232,133 @@ step_1a_and_step_1b_combined_python_pandas:
   steps:
   - step_1a
   - step_1b
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
+  image_name: python_pandas.sif
+  zenodo_record_id: 15611084
+  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
 dummy_step_1_for_output_dir_example:
   steps:
   - step_1_for_output_dir_example
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
+  image_name: main/dummy_step_1_for_output_dir_example.sif
   script_cmd: python /dummy_step_1_for_output_dir_example.py
   outputs:
     step_1_main_output_directory: output_dir/
 dummy_step_1_for_output_dir_example_default:
   steps:
   - step_1_for_output_dir_example
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
+  image_name: main/dummy_step_1_for_output_dir_example.sif
   script_cmd: python /dummy_step_1_for_output_dir_example.py
 dummy_step_2_for_output_dir_example:
   steps:
   - step_2_for_output_dir_example
-  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_2_for_output_dir_example.sif
+  image_name: main/dummy_step_2_for_output_dir_example.sif
   script_cmd: python /dummy_step_2_for_output_dir_example.py
   outputs:
     step_2_main_output: result.parquet
+default_removing_records:
+  steps:
+  - removing_records
+  image_name: main/default_removing_records.sif
+  script_cmd: python /default_removing_records.py
+  outputs:
+    dataset: dataset
+default_clusters_to_links:
+  steps:
+  - clusters_to_links
+  image_name: main/default_clusters_to_links.sif
+  script_cmd: python /default_clusters_to_links.py
+  outputs:
+    known_links: result.parquet
+default_determining_exclusions:
+  steps:
+  - determining_exclusions
+  image_name: main/default_determining_exclusions.sif
+  script_cmd: python /default_determining_exclusions.py
+  outputs:
+    ids_to_remove: result.parquet
+default_updating_clusters:
+  steps:
+  - updating_clusters
+  image_name: main/default_updating_clusters.sif
+  script_cmd: python /default_updating_clusters.py
+  outputs:
+    clusters: clusters.parquet
+dummy_canonicalizing_and_downstream_analysis:
+  steps:
+  - canonicalizing_and_downstream_analysis
+  image_name: main/dummy_canonicalizing_and_downstream_analysis.sif
+  script_cmd: python /dummy_canonicalizing_and_downstream_analysis.py
+  outputs:
+    analysis_output: result.parquet
+dummy_pre-processing:
+  steps:
+  - pre-processing
+  image_name: main/dummy_pre-processing.sif
+  script_cmd: python /dummy_pre-processing.py
+  outputs:
+    dataset: dataset
+default_schema_alignment:
+  steps:
+  - schema_alignment
+  image_name: main/default_schema_alignment.sif
+  script_cmd: python /default_schema_alignment.py
+  outputs:
+    records: result.parquet
+splink_blocking_and_filtering:
+  steps:
+  - blocking_and_filtering
+  image_name: main/splink_blocking_and_filtering.sif
+  script_cmd: python /splink_blocking_and_filtering.py
+  outputs:
+    blocks: blocks
+splink_evaluating_pairs:
+  steps:
+  - evaluating_pairs
+  image_name: main/splink_evaluating_pairs.sif
+  script_cmd: python /splink_evaluating_pairs.py
+  outputs:
+    links: result.parquet
+splink_links_to_clusters:
+  steps:
+  - links_to_clusters
+  image_name: main/splink_links_to_clusters.sif
+  script_cmd: python /splink_links_to_clusters.py
+  outputs:
+    clusters: result.parquet
+fastLink_evaluating_pairs:
+  steps:
+  - evaluating_pairs
+  image_name: main/fastLink_evaluating_pairs.sif
+  script_cmd: Rscript /fastLink_evaluating_pairs.R
+  outputs:
+    links: result.parquet
+fastLink_links_to_clusters:
+  steps:
+  - links_to_clusters
+  image_name: main/fastLink_links_to_clusters.sif
+  script_cmd: Rscript /fastLink_links_to_clusters.R
+  outputs:
+    clusters: result.parquet
+exclude_clustered:
+  steps:
+  - determining_exclusions
+  image_name: main/exclude_clustered.sif
+  script_cmd: python /exclude_clustered.py
+  outputs:
+    ids_to_remove: result.parquet
+exclude_none:
+  steps:
+  - determining_exclusions
+  image_name: main/exclude_none.sif
+  script_cmd: python /exclude_none.py
+  outputs:
+    ids_to_remove: result.parquet
+update_clusters_by_connected_components:
+  steps:
+  - updating_clusters
+  image_name: main/update_clusters_by_connected_components.sif
+  script_cmd: python /update_clusters_by_connected_components.py
+  outputs:
+    clusters: result.parquet

easylink/pipeline.py

@@ -25,7 +25,6 @@ from easylink.rule import (
 )
 from easylink.utilities.general_utils import exit_with_validation_error
 from easylink.utilities.paths import SPARK_SNAKEFILE
-from easylink.utilities.validation_utils import validate_input_file_dummy
 
 IMPLEMENTATION_ERRORS_KEY = "IMPLEMENTATION ERRORS"
 
@@ -135,7 +134,10 @@ class Pipeline:
         """
         errors = defaultdict(dict)
         for implementation in self.pipeline_graph.implementations:
-            implementation_errors = implementation.validate()
+            implementation_errors = implementation.validate(
+                skip_image_validation=(self.config.command == "generate_dag"),
+                images_dir=self.config.images_dir,
+            )
             if implementation_errors:
                 errors[IMPLEMENTATION_ERRORS_KEY][implementation.name] = implementation_errors
         return errors
@@ -206,7 +208,14 @@ wildcard_constraints:
         The input files to the target rule (i.e. the result node) are the final
         output themselves.
         """
-        final_output, _ = self.pipeline_graph.get_io_filepaths("results")
+        input_slots, _ = self.pipeline_graph.get_io_slot_attributes("results")
+
+        if len(input_slots) != 1:
+            raise ValueError("Results node must have only one input slot")
+
+        input_slot_name = list(input_slots.keys())[0]
+        input_slot_attrs = input_slots[input_slot_name]
+        final_output = input_slot_attrs["filepaths"]
         validator_file = str("input_validations/final_validator")
         # Snakemake resolves the DAG based on the first rule, so we put the target
         # before the validation
@@ -217,10 +226,10 @@ wildcard_constraints:
         )
         final_validation = InputValidationRule(
             name="results",
-            input_slot_name="main_input",
+            input_slot_name=input_slot_name,
             input=final_output,
             output=validator_file,
-            validator=validate_input_file_dummy,
+            validator=input_slot_attrs["validator"],
         )
         target_rule.write_to_snakefile(self.snakefile_path)
         final_validation.write_to_snakefile(self.snakefile_path)
@@ -322,7 +331,7 @@ use rule start_spark_worker from spark_cluster with:
             resources=resources,
             envvars=implementation.environment_variables,
             diagnostics_dir=str(diagnostics_dir),
-            image_path=implementation.singularity_image_path,
+            image_path=self.config.images_dir / implementation.singularity_image_name,
             script_cmd=implementation.script_cmd,
             requires_spark=implementation.requires_spark,
             is_embarrassingly_parallel=is_embarrassingly_parallel,

easylink/pipeline_schema_constants/__init__.py

@@ -9,10 +9,10 @@ package defines the nodes and edges required to instantiate such ``PipelineSchem
 
 """
 
-from easylink.pipeline_schema_constants import development, testing
+from easylink.pipeline_schema_constants import development, main, testing
 
 SCHEMA_PARAMS = {
-    "main": "TODO",
+    "main": main.SCHEMA_PARAMS,
     # development and testing
     "development": development.SCHEMA_PARAMS,
     "integration": testing.SCHEMA_PARAMS_ONE_STEP,