easylink 0.1.21__py3-none-any.whl → 0.1.23__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.21"
+__version__ = "0.1.23"

easylink/configuration.py

@@ -184,7 +184,9 @@ class Config(LayeredConfigTree):
     #################
 
     def _get_schema(self, schema_name: str = "main") -> PipelineSchema:
-        """Returns the first :class:`~easylink.pipeline_schema.PipelineSchema` that validates the requested pipeline.
+        """Gets the requested :class:`~easylink.pipeline_schema.PipelineSchema`.
+
+        The schema is only returned if it validates the pipeline configuration.
 
         Parameters
         ----------
@@ -205,11 +207,10 @@ class Config(LayeredConfigTree):
         Notes
         -----
         This acts as the pipeline configuration file's validation method since
-        we can only find a matching ``PipelineSchema`` if that file is valid.
+        we can only validate the ``PipelineSchema`` if that file is valid.
 
         """
         errors = defaultdict(dict)
-        # Try each schema until one is validated
         schema = PipelineSchema.get_schema(schema_name)
         logs = schema.validate_step(self.pipeline, self.input_data)
         if logs:

easylink/implementation_metadata.yaml

@@ -2,8 +2,8 @@ step_1_python_pandas:
   steps:
   - step_1
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -11,8 +11,8 @@ step_1a_python_pandas:
   steps:
   - step_1a
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -22,8 +22,8 @@ step_1b_python_pandas:
   steps:
   - step_1b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -33,8 +33,8 @@ step_2_python_pandas:
   steps:
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -42,8 +42,8 @@ step_3_python_pandas:
   steps:
   - step_3
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_3_main_output: result.parquet
@@ -51,8 +51,8 @@ step_4_python_pandas:
   steps:
   - step_4
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -62,8 +62,8 @@ step_5_python_pandas:
   steps:
   - step_5
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -73,8 +73,8 @@ step_6_python_pandas:
   steps:
   - step_6
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -84,8 +84,8 @@ step_4a_python_pandas:
   steps:
   - step_4a
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -95,8 +95,8 @@ step_4b_python_pandas:
   steps:
   - step_4b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -106,8 +106,8 @@ step_4b_r:
   steps:
   - step_4b
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -117,8 +117,8 @@ step_1_python_pyspark:
   steps:
   - step_1
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -127,8 +127,8 @@ step_2_python_pyspark:
   steps:
   - step_2
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -137,8 +137,8 @@ step_3_python_pyspark:
   steps:
   - step_3
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_3_main_output: result.parquet
@@ -147,8 +147,8 @@ step_4_python_pyspark:
   steps:
   - step_4
   image_name: python_pyspark.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 6fb2a2119630138f4db82356b8d78b87
+  zenodo_record_id: 15733426
+  md5_checksum: c948577ab0607411dd4b640622d9ec3a
   script_cmd: python3 /code/dummy_step.py
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -158,8 +158,8 @@ step_1_r:
   steps:
   - step_1
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_1_main_output: result.parquet
@@ -168,8 +168,8 @@ step_2_r:
   steps:
   - step_2
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_2_main_output: result.parquet
@@ -178,8 +178,8 @@ step_3_r:
   steps:
   - step_3
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_3_main_output: result.parquet
@@ -188,8 +188,8 @@ step_4_r:
   steps:
   - step_4
   image_name: r-image.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 9410af1317aabc332604cbec33b59d42
+  zenodo_record_id: 15733426
+  md5_checksum: a4a03b836694a2b81a1bd2852736ccc5
   script_cmd: Rscript /dummy_step.R
   env:
     INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
@@ -201,8 +201,8 @@ step_1_and_step_2_combined_python_pandas:
   - step_1
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
@@ -211,8 +211,8 @@ step_1_and_step_2_parallel_python_pandas:
   - step_1
   - step_2
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   env:
     INPUT_ENV_VARS: STEP_1_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,STEP_2_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS
@@ -223,8 +223,8 @@ step_3_and_step_4_combined_python_pandas:
   - step_3
   - step_4
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_4_main_output: result.parquet
@@ -233,8 +233,8 @@ step_1a_and_step_1b_combined_python_pandas:
   - step_1a
   - step_1b
   image_name: python_pandas.sif
-  zenodo_record_id: 15611084
-  md5_checksum: 7cc7cb37195c635684903b6777cf1cdf
+  zenodo_record_id: 15733426
+  md5_checksum: 9177b8e168fcc9cae91bf61265f2185c
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
@@ -362,3 +362,10 @@ update_clusters_by_connected_components:
   script_cmd: python /update_clusters_by_connected_components.py
   outputs:
     clusters: result.parquet
+middle_name_to_initial:
+  steps:
+  - pre-processing
+  image_name: main/middle_name_to_initial.sif
+  script_cmd: python /middle_name_to_initial.py
+  outputs:
+    dataset: dataset

easylink/pipeline_schema.py

@@ -159,10 +159,10 @@ class PipelineSchema(HierarchicalStep):
         )
 
     @classmethod
-    def get_schema(cls, name: str = "main") -> list["PipelineSchema"]:
-        """Gets all allowable ``PipelineSchemas``.
+    def get_schema(cls, name: str = "main") -> "PipelineSchema":
+        """Gets the requested ``PipelineSchema``.
 
-        These ``PipelineSchemas`` represent the fully supported pipelines and are
+        This ``PipelineSchema`` represents the fully supported pipelines and is
         used to validate the user-requested pipeline.
 
         Parameters

easylink/pipeline_schema_constants/__init__.py

@@ -27,4 +27,5 @@ SCHEMA_PARAMS = {
     "auto_parallel_cloneable_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_CLONEABLE_STEP,
     "auto_parallel_loop_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_LOOP_STEP,
     "auto_parallel_hierarchical_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_HIERARCHICAL_STEP,
+    "default_implementations": testing.SCHEMA_PARAMS_DEFAULT_IMPLEMENTATIONS,
 }

easylink/pipeline_schema_constants/testing.py

@@ -607,7 +607,7 @@ NODES_OUTPUT_DIR = [
         input_slots=[
             InputSlot(
                 name="step_2_main_input",
-                env_var="DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH",
+                env_var="MAIN_INPUT_DIR_PATH",
                 validator=validate_dir,
             )
         ],
@@ -640,3 +640,126 @@ EDGES_OUTPUT_DIR = [
     ),
 ]
 SCHEMA_PARAMS_OUTPUT_DIR = (NODES_OUTPUT_DIR, EDGES_OUTPUT_DIR)
+
+
+NODES_DEFAULT_IMPLEMENTATIONS = [
+    InputStep(),
+    HierarchicalStep(
+        step_name="step_1",
+        input_slots=[
+            InputSlot(
+                name="step_1_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            ),
+        ],
+        output_slots=[OutputSlot("step_1_main_output")],
+        nodes=[
+            Step(
+                step_name="step_1a",
+                input_slots=[
+                    InputSlot(
+                        name="step_1a_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                    ),
+                ],
+                output_slots=[OutputSlot("step_1a_main_output")],
+                default_implementation="step_1a_python_pandas",
+            ),
+            Step(
+                step_name="step_1b",
+                input_slots=[
+                    InputSlot(
+                        name="step_1b_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                    ),
+                ],
+                output_slots=[OutputSlot("step_1b_main_output")],
+                default_implementation="step_1b_python_pandas",
+            ),
+        ],
+        edges=[
+            EdgeParams(
+                source_node="step_1a",
+                target_node="step_1b",
+                output_slot="step_1a_main_output",
+                input_slot="step_1b_main_input",
+            ),
+        ],
+        input_slot_mappings=[
+            InputSlotMapping(
+                parent_slot="step_1_main_input",
+                child_node="step_1a",
+                child_slot="step_1a_main_input",
+            ),
+        ],
+        output_slot_mappings=[
+            OutputSlotMapping(
+                parent_slot="step_1_main_output",
+                child_node="step_1b",
+                child_slot="step_1b_main_output",
+            ),
+        ],
+        default_implementation="step_1_python_pandas",
+    ),
+    Step(
+        step_name="step_2",
+        input_slots=[
+            InputSlot(
+                name="step_2_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            )
+        ],
+        output_slots=[OutputSlot("step_2_main_output")],
+        default_implementation="step_2_python_pandas",
+    ),
+    LoopStep(
+        template_step=Step(
+            step_name="step_3",
+            input_slots=[
+                InputSlot(
+                    name="step_3_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                )
+            ],
+            output_slots=[OutputSlot("step_3_main_output")],
+        ),
+        self_edges=[
+            EdgeParams(
+                source_node="step_3",
+                target_node="step_3",
+                output_slot="step_3_main_output",
+                input_slot="step_3_main_input",
+            ),
+        ],
+        default_implementation="step_3_python_pandas",
+    ),
+    CloneableStep(
+        template_step=Step(
+            step_name="step_4",
+            input_slots=[
+                InputSlot(
+                    name="step_4_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                ),
+            ],
+            output_slots=[
+                OutputSlot(
+                    name="step_4_main_output",
+                ),
+            ],
+        ),
+        default_implementation="step_4_python_pandas",
+    ),
+    OutputStep(
+        input_slots=[
+            InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
+        ],
+    ),
+]
+SCHEMA_PARAMS_DEFAULT_IMPLEMENTATIONS = (NODES_DEFAULT_IMPLEMENTATIONS, EDGES_TWO_STEPS)

easylink/rule.py

@@ -182,15 +182,15 @@ rule:
         # TODO [MIC-5787]: handle multiple wildcards, e.g.
         #   output_paths = ",".join(self.output)
         #   wildcards_subdir = "/".join([f"{{wildcards.{wc}}}" for wc in self.wildcards])
-        #   and then in shell cmd: export DUMMY_CONTAINER_OUTPUT_PATHS={output_paths}/{wildcards_subdir}
+        #   and then in shell cmd: export OUTPUT_PATHS={output_paths}/{wildcards_subdir}
 
         # snakemake shell commands require wildcards to be prefaced with 'wildcards.'
         output_files = ",".join(self.output).replace("{chunk}", "{wildcards.chunk}")
         shell_cmd = f"""
     shell:
         '''
-        export DUMMY_CONTAINER_OUTPUT_PATHS={output_files}
-        export DUMMY_CONTAINER_DIAGNOSTICS_DIRECTORY={self.diagnostics_dir}"""
+        export OUTPUT_PATHS={output_files}
+        export DIAGNOSTICS_DIRECTORY={self.diagnostics_dir}"""
         for input_slot_attrs in self.input_slots.values():
             # snakemake shell commands require wildcards to be prefaced with 'wildcards.'
             input_files = ",".join(input_slot_attrs["filepaths"]).replace(
@@ -200,8 +200,8 @@ rule:
         export {input_slot_attrs["env_var"]}={input_files}"""
         if self.requires_spark:
             shell_cmd += f"""
-        read -r DUMMY_CONTAINER_SPARK_MASTER_URL < {{input.master_url}}
-        export DUMMY_CONTAINER_SPARK_MASTER_URL"""
+        read -r SPARK_MASTER_URL < {{input.master_url}}
+        export  SPARK_MASTER_URL"""
         for var_name, var_value in self.envvars.items():
             shell_cmd += f"""
         export {var_name}={var_value}"""

easylink/step.py

@@ -92,6 +92,7 @@ class Step:
         input_slot_mappings: Iterable[InputSlotMapping] = (),
         output_slot_mappings: Iterable[OutputSlotMapping] = (),
         is_auto_parallel: bool = False,
+        default_implementation: str | None = None,
     ) -> None:
         if not step_name and not name:
             raise ValueError("All Steps must contain a step_name, name, or both.")
@@ -127,6 +128,9 @@ class Step:
         ``OutputSlotMappings`` of this ``Step``."""
         self.is_auto_parallel = is_auto_parallel
         """Whether or not this ``Step`` is to be automatically run in parallel."""
+        self.default_implementation = default_implementation
+        """The default implementation to use for this ``Step`` if the ``Step`` is
+        not explicitly configured in the pipeline specification."""
         self.parent_step = None
         """This ``Step's`` parent ``Step``, if applicable."""
         self._configuration_state = None
@@ -580,6 +584,7 @@ class HierarchicalStep(Step):
         input_slot_mappings=(),
         output_slot_mappings=(),
         directly_implemented=True,
+        default_implementation: str | None = None,
     ):
         super().__init__(
             step_name,
@@ -588,6 +593,7 @@ class HierarchicalStep(Step):
             output_slots,
             input_slot_mappings,
             output_slot_mappings,
+            default_implementation=default_implementation,
         )
         self.nodes = nodes
         """All sub-nodes (i.e. sub-``Steps``) that make up this ``HierarchicalStep``."""
@@ -722,13 +728,19 @@ class HierarchicalStep(Step):
             step = self.step_graph.nodes[node]["step"]
             if isinstance(step, IOStep):
                 continue
+            if step.name not in step_config:
+                default_implementation = self.step_graph.nodes[step.name][
+                    "step"
+                ].default_implementation
+                step_errors = (
+                    {f"step {step.name}": ["The step is not configured."]}
+                    if not default_implementation
+                    else {}
+                )
             else:
-                if step.name not in step_config:
-                    step_errors = {f"step {step.name}": ["The step is not configured."]}
-                else:
-                    step_errors = step.validate_step(
-                        step_config[step.name], combined_implementations, input_data_config
-                    )
+                step_errors = step.validate_step(
+                    step_config[step.name], combined_implementations, input_data_config
+                )
             if step_errors:
                 errors.update(step_errors)
         extra_steps = set(step_config.keys()) - set(self.step_graph.nodes)
@@ -830,12 +842,14 @@ class TemplatedStep(Step, ABC):
     def __init__(
         self,
         template_step: Step,
+        default_implementation: str | None = None,
     ) -> None:
         super().__init__(
             template_step.step_name,
             template_step.name,
             template_step.input_slots.values(),
             template_step.output_slots.values(),
+            default_implementation=default_implementation,
         )
         self.step_graph = None
         """The :class:`~easylink.graph_components.StepGraph` i.e. the directed acyclic 
@@ -1110,8 +1124,9 @@ class LoopStep(TemplatedStep):
         self,
         template_step: Step | None = None,
         self_edges: Iterable[EdgeParams] = (),
+        default_implementation: str | None = None,
     ) -> None:
-        super().__init__(template_step)
+        super().__init__(template_step, default_implementation)
         self.self_edges = self_edges
         """:class:`~easylink.graph_components.EdgeParams` that represent self-edges,
         i.e. edges that connect the output of one loop to the input of the next."""
@@ -1119,7 +1134,7 @@ class LoopStep(TemplatedStep):
     @property
     def config_key(self):
         """The pipeline specification key required for a ``LoopStep``."""
-        return "iterate"
+        return "iterations"
 
     @property
     def node_prefix(self):
@@ -2181,15 +2196,32 @@ class NonLeafConfigurationState(ConfigurationState):
 
         This method recursively traverses the ``StepGraph`` and sets the configuration
         state for each ``Step`` until reaching all leaf nodes.
+
+        Notes
+        -----
+        If a ``Step`` name is missing from the ``step_config``, we know that it
+        must have a default implementation because we already validated that one
+        exists during :meth:`HierarchicalStep._validate_step_graph`. In that case,
+        we manually instantiate and use a ``step_config`` with the default implementation.
         """
         for sub_node in self._step.step_graph.nodes:
             sub_step = self._step.step_graph.nodes[sub_node]["step"]
-            # IOSteps, SplitterSteps, and AggregatorSteps never appear explicitly in the configuration
-            step_config = (
-                self.step_config
-                if isinstance(sub_step, (IOStep, SplitterStep, AggregatorStep))
-                else self.step_config[sub_step.name]
-            )
+            try:
+                step_config = (
+                    self.step_config
+                    if isinstance(sub_step, StandaloneStep)
+                    else self.step_config[sub_step.name]
+                )
+            except KeyError:
+                # We know that any missing keys must have a default implementation
+                # (because we have already checked that it exists during validation)
+                step_config = LayeredConfigTree(
+                    {
+                        "implementation": {
+                            "name": sub_step.default_implementation,
+                        }
+                    }
+                )
             sub_step.set_configuration_state(
                 step_config, self.combined_implementations, self.input_data_config
             )

easylink/steps/cascading/exclude_clustered.py

@@ -69,8 +69,8 @@ clustered_record_ids = set(dataset_df["Record ID"].unique()) & set(
 
 IDS_TO_REMOVE = pd.DataFrame({"Record ID": list(clustered_record_ids)})
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/exclude_none.py

@@ -69,8 +69,8 @@ clusters_df = load_file(clusters_filepath)
 
 IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/update_clusters_by_connected_components.py

@@ -50,8 +50,8 @@ if len(known_clusters_filepaths) == 0:
 known_clusters_filepath = known_clusters_filepaths[0]
 known_clusters_df = load_file(known_clusters_filepath)
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (clusters.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (clusters.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 Path(results_filepath).parent.mkdir(exist_ok=True, parents=True)
 
 new_clusters_df = load_file(new_clusters_filepath)

easylink/steps/default/default_clusters_to_links.py

@@ -80,8 +80,8 @@ if len(clusters_filepaths) == 0:
 
 clusters_filepath = clusters_filepaths[0]
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a path to a single file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 clusters_df = load_file(clusters_filepath)
 links_df = clusters_to_links(clusters_df)

easylink/steps/default/default_determining_exclusions.py

@@ -74,8 +74,8 @@ if len(clusters_df) > 0:
 
 IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
 
-# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
-results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+# OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["OUTPUT_PATHS"]
 
 logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
 IDS_TO_REMOVE.to_parquet(results_filepath)