easylink 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.4"
+__version__ = "0.1.6"

easylink/configuration.py

@@ -97,7 +97,7 @@ class Config(LayeredConfigTree):
     def __init__(
         self,
         config_params: dict[str, Any],
-        potential_schemas: list[PipelineSchema] | PipelineSchema = PIPELINE_SCHEMAS,
+        potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
     ) -> None:
         super().__init__(layers=["initial_data", "default", "user_configured"])
         self.update(DEFAULT_ENVIRONMENT, layer="default")

easylink/pipeline_schema_constants/__init__.py

@@ -1,11 +1,12 @@
-from easylink.pipeline_schema_constants import development, tests
+from easylink.pipeline_schema_constants import development, testing
 
 ALLOWED_SCHEMA_PARAMS = {
     "development": development.SCHEMA_PARAMS,
 }
 
 TESTING_SCHEMA_PARAMS = {
-    "integration": tests.SINGLE_STEP_SCHEMA_PARAMS,
-    "combined_bad_topology": tests.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
-    "combined_bad_implementation_names": tests.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
+    "integration": testing.SINGLE_STEP_SCHEMA_PARAMS,
+    "combine_bad_topology": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
+    "combine_bad_implementation_names": testing.BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS,
+    "nested_templated_steps": testing.NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS,
 }

easylink/pipeline_schema_constants/{tests.py → testing.py}

@@ -11,7 +11,14 @@ from easylink.graph_components import (
     OutputSlot,
     OutputSlotMapping,
 )
-from easylink.step import HierarchicalStep, InputStep, LoopStep, OutputStep, Step
+from easylink.step import (
+    HierarchicalStep,
+    InputStep,
+    LoopStep,
+    OutputStep,
+    ParallelStep,
+    Step,
+)
 from easylink.utilities.validation_utils import validate_input_file_dummy
 
 SINGLE_STEP_NODES = [
@@ -128,3 +135,85 @@ BAD_COMBINED_TOPOLOGY_NODES = [
 ]
 
 BAD_COMBINED_TOPOLOGY_SCHEMA_PARAMS = (BAD_COMBINED_TOPOLOGY_NODES, SINGLE_STEP_EDGES)
+
+
+NESTED_TEMPLATED_STEPS_NODES = [
+    InputStep(),
+    LoopStep(
+        template_step=ParallelStep(
+            template_step=HierarchicalStep(
+                step_name="step_1",
+                input_slots=[
+                    InputSlot(
+                        name="step_1_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                    ),
+                ],
+                output_slots=[OutputSlot("step_1_main_output")],
+                nodes=[
+                    Step(
+                        step_name="step_1a",
+                        input_slots=[
+                            InputSlot(
+                                name="step_1a_main_input",
+                                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                validator=validate_input_file_dummy,
+                            ),
+                        ],
+                        output_slots=[OutputSlot("step_1a_main_output")],
+                    ),
+                    Step(
+                        step_name="step_1b",
+                        input_slots=[
+                            InputSlot(
+                                name="step_1b_main_input",
+                                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                validator=validate_input_file_dummy,
+                            ),
+                        ],
+                        output_slots=[OutputSlot("step_1b_main_output")],
+                    ),
+                ],
+                edges=[
+                    EdgeParams(
+                        source_node="step_1a",
+                        target_node="step_1b",
+                        output_slot="step_1a_main_output",
+                        input_slot="step_1b_main_input",
+                    ),
+                ],
+                input_slot_mappings=[
+                    InputSlotMapping(
+                        parent_slot="step_1_main_input",
+                        child_node="step_1a",
+                        child_slot="step_1a_main_input",
+                    ),
+                ],
+                output_slot_mappings=[
+                    OutputSlotMapping(
+                        parent_slot="step_1_main_output",
+                        child_node="step_1b",
+                        child_slot="step_1b_main_output",
+                    ),
+                ],
+            ),
+        ),
+        self_edges=[
+            EdgeParams(
+                source_node="step_1",
+                target_node="step_1",
+                output_slot="step_1_main_output",
+                input_slot="step_1_main_input",
+            ),
+        ],
+    ),
+    OutputStep(
+        input_slots=[
+            InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
+        ],
+    ),
+]
+
+
+NESTED_TEMPLATED_STEPS_SCHEMA_PARAMS = (NESTED_TEMPLATED_STEPS_NODES, SINGLE_STEP_EDGES)

easylink/step.py

@@ -282,6 +282,17 @@ class Step:
     ) -> None:
         """Sets the configuration state for this ``Step``.
 
+        The so-called 'configuration state' for a given ``Step`` is backed up by
+        a :class:`ConfigurationState` class and is assigned to its :attr:`_configuration_state`
+        attribute. There are two possible ``ConfigurationStates``:
+        :class:`LeafConfigurationState` and :class:`NonLeafConfigurationState`.
+
+        This method sets the configuration state of this ``Step`` based on whether
+        or not a :attr:`config_key` is set *and exists is the ``Step's`` configuration*
+        (i.e. its portion of the user-suppled pipeline specification
+        file); any required deviation from this behavior requires special
+        handling.
+
         Parameters
         ----------
         parent_config
@@ -378,8 +389,9 @@ class Step:
     ) -> dict[str, list[str]]:
         """Validates a non-leaf ``Step``."""
         errors = {}
-        for node in self.step_graph.nodes:
-            step = self.step_graph.nodes[node]["step"]
+        nodes = self.step_graph.nodes
+        for node in nodes:
+            step = nodes[node]["step"]
             if isinstance(step, IOStep):
                 continue
             if step.name not in step_config:
@@ -390,7 +402,7 @@ class Step:
                 )
             if step_errors:
                 errors.update(step_errors)
-        extra_steps = set(step_config.keys()) - set(self.step_graph.nodes)
+        extra_steps = set(step_config.keys()) - set(nodes)
         for extra_step in extra_steps:
             errors[f"step {extra_step}"] = [f"{extra_step} is not a valid step."]
         return errors
@@ -807,12 +819,42 @@ class TemplatedStep(Step, ABC):
             The configuration for any implementations to be combined.
         input_data_config
             The input data configuration for the entire pipeline.
+
+        Notes
+        -----
+        A ``TemplatedStep`` is *always* assigned a :class:`NonLeafConfigurationState`
+        even if it has no multiplicity since (despite having no copies to make) we
+        still need to traverse the sub-``Steps`` to get to the one with a single
+        :class:`~easylink.implementation.Implementation`, i.e. the one with a
+        :class:`LeafConfigurationState`.
         """
-        num_repeats = len(self._get_config(parent_config[self.name]))
-        self.step_graph = self._update_step_graph(num_repeats)
-        self.slot_mappings = self._update_slot_mappings(num_repeats)
-        super().set_configuration_state(
-            parent_config, combined_implementations, input_data_config
+        step_config = parent_config[self.name]
+        if self.config_key not in step_config:
+            # Special handle the step_graph update
+            self.step_graph = StepGraph()
+            self.template_step.name = self.name
+            self.step_graph.add_node_from_step(self.template_step)
+            # Special handle the slot_mappings update
+            input_mappings = [
+                InputSlotMapping(slot, self.name, slot) for slot in self.input_slots
+            ]
+            output_mappings = [
+                OutputSlotMapping(slot, self.name, slot) for slot in self.output_slots
+            ]
+            self.slot_mappings = {"input": input_mappings, "output": output_mappings}
+            # Add the key back to the expanded config
+            expanded_config = LayeredConfigTree({self.name: step_config})
+        else:
+            expanded_config = self._get_config(step_config)
+            num_repeats = len(expanded_config)
+            self.step_graph = self._update_step_graph(num_repeats)
+            self.slot_mappings = self._update_slot_mappings(num_repeats)
+        # Manually set the configuration state to non-leaf instead of relying
+        # on super().get_configuration_state() because that method will erroneously
+        # set to leaf state in the event the user didn't include the config_key
+        # in the pipeline specification.
+        self._configuration_state = NonLeafConfigurationState(
+            self, expanded_config, combined_implementations, input_data_config
         )
 
     def _duplicate_template_step(self) -> Step:
@@ -1105,9 +1147,10 @@ class ChoiceStep(Step):
         initial ones are handled.
 
         We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
-        here as opposed to in :meth:`set_configuration_state` (as is done in :class:`TemplatedStep`)
-        because ``ChoiceStep`` validation happens prior to setting the configuration
-        state and actually requires the ``StepGraph`` and ``SlotMappings``.
+        in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
+        as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
+        prior to :meth:`set_configuration_state`, but the validations itself actually
+        requires the updated ``StepGraph`` and ``SlotMappings``.
 
         We do not attempt to validate the subgraph here if the 'type' key is unable
         to be validated.
@@ -1136,7 +1179,7 @@ class ChoiceStep(Step):
                 ]
             }
 
-        # Handle the actual chosen step_config
+        # HACK: Update the step graph and mappings here because we need them for validation
         self.step_graph = self._update_step_graph(subgraph)
         self.slot_mappings = self._update_slot_mappings(subgraph)
         # NOTE: A ChoiceStep is by definition non-leaf step
@@ -1163,11 +1206,11 @@ class ChoiceStep(Step):
 
         Notes
         -----
-        We update the :class:`~easylink.graph_components.StepGraph` and
-        :class:`SlotMappings<easylink.graph_components.SlotMapping>` in
-        :meth:`validate_step` as opposed to here (as is done with
-        :class:`TemplatedSteps<TemplatedStep>`) because ``ChoiceStep`` validation
-        happens prior to this but requires the ``StepGraph`` and ``SlotMappings``.
+        We update the :class:`easylink.graph_components.StepGraph` and ``SlotMappings``
+        in :meth:`validate_step` (as opposed to in :meth:`set_configuration_state`
+        as is done in :class:`TemplatedStep`) because :meth:`validate_step` is called
+        prior to :meth:`set_configuration_state`, but the validations itself actually
+        requires the updated ``StepGraph`` and ``SlotMappings``.
         """
 
         chosen_parent_config = LayeredConfigTree(
@@ -1364,7 +1407,6 @@ class LeafConfigurationState(ConfigurationState):
             for mapping in mappings:
                 imp_edge = mapping.remap_edge(edge)
                 implementation_edges.append(imp_edge)
-
         elif edge.target_node == self._step.name:
             mappings = [
                 mapping
@@ -1520,7 +1562,6 @@ class NonLeafConfigurationState(ConfigurationState):
                 new_step = self._step.step_graph.nodes[mapping.child_node]["step"]
                 imp_edges = new_step.get_implementation_edges(new_edge)
                 implementation_edges.extend(imp_edges)
-
         elif edge.target_node == self._step.name:
             mappings = [
                 mapping
@@ -1544,8 +1585,9 @@ class NonLeafConfigurationState(ConfigurationState):
         This method recursively traverses the ``StepGraph`` and sets the configuration
         state for each ``Step`` until reaching all leaf nodes.
         """
-        for node in self._step.step_graph.nodes:
-            step = self._step.step_graph.nodes[node]["step"]
+        nodes = self._step.step_graph.nodes
+        for node in nodes:
+            step = nodes[node]["step"]
             step.set_configuration_state(
                 self.pipeline_config, self.combined_implementations, self.input_data_config
             )

{easylink-0.1.4.dist-info → easylink-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: easylink
-Version: 0.1.4
+Version: 0.1.6
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.4.dist-info → easylink-0.1.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=Wzf5T3NBDfhQoTnhnRNHSlAsE0XMqbclXG-M81Vas70,22
+easylink/_version.py,sha256=n3oM6B_EMz93NsTI18NNZd-jKFcUPzUkbIKj5VFK5ok,22
 easylink/cli.py,sha256=O0YvFnjxM3N8ox6wMi5-SSQZ6bS4_VcRT3apCIY7u0I,6117
-easylink/configuration.py,sha256=raF7-sRE4ImM85LIyetNcrMy0LXWGLkfBD58T_BowJM,12530
+easylink/configuration.py,sha256=Ire2pMZNZ6wtSwhcWnQpYa-snX4KrhXgovlQwQ2Wxf4,12530
 easylink/graph_components.py,sha256=LIHLrASrppTr9XHUFZymtT9rE7_cFzCB1LVxyO3pCWk,11554
 easylink/implementation.py,sha256=1TFbsUxOkDkyyMtCwYQtA6uwZb5TawEw4CHjjSot0_s,8873
 easylink/implementation_metadata.yaml,sha256=VvlEu3Dvlmeh1MpzeYx91j22GiV-9mu3hZP5yVuW04o,6763
@@ -11,12 +11,12 @@ easylink/pipeline_graph.py,sha256=K8padPqQG6kSupYCQgbjYn_gs51tE1CmOvWmZ49EKLc,19
 easylink/pipeline_schema.py,sha256=ckvA4deRYalY5dLLbJDrO_pKttMuWnEUvSn5fSdu4jc,5900
 easylink/rule.py,sha256=dHr95tI4O39makPp9nEGFaIsGhOoa93RwuVzIVXUhak,7606
 easylink/runner.py,sha256=CSqYDWzY4pBvaklUUvj75UeJ4VxqwW9MYgcwGrAlspo,6222
-easylink/step.py,sha256=bui2J_V9J8Is2RUliOTacUHEJ01OyEA91oGL0au8_9I,60324
+easylink/step.py,sha256=NCN1L5ojpfJ1CgV_Ih4duQL_aUFL3ri_XiNguH20JDE,62709
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=APGQ8iHgDgi76HByVRRM1cUfs0Q1uF1SwgbNglh60Is,378
+easylink/pipeline_schema_constants/__init__.py,sha256=gVg8_0xv4FxG3B-Jy5MwiXIVntu4m36uCdA3ohVTN7w,460
 easylink/pipeline_schema_constants/development.py,sha256=WaS6EQzgCI6opnY-7OCHpT0JpP2OvRmYrWVdt_ND_bk,10915
-easylink/pipeline_schema_constants/tests.py,sha256=Vtj-1CB_uaEEQaZN0HZ79A8cKPQva9_fps-HfvM97Hw,3989
+easylink/pipeline_schema_constants/testing.py,sha256=GUJtVGIzLd71j7hX_qaSuMT3FCxwFOGzZwWc_1Tbtsk,7016
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
 easylink/steps/dev/build-containers-remote.sh,sha256=Hy-kaaXf-ta6n8SzOz_ahByjMY5T7J71MvzXRXDvQw8,271
@@ -41,8 +41,8 @@ easylink/utilities/general_utils.py,sha256=PSlTiHmafuI_dDSH0cW7iSCkTV0WgqX9ZgzlT
 easylink/utilities/paths.py,sha256=yl0cuWChJmB6YKMCQavTKw9jIl-VQhH6cnsM6D5c0Zk,599
 easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
 easylink/utilities/validation_utils.py,sha256=qOgn1n3_m5blFN7eHJ9MbOt5DkFA6DWucAOUAjvGvco,764
-easylink-0.1.4.dist-info/METADATA,sha256=rmjAl2vMMuK3pED9MWxpwDJbA39loKO6EZPAnsxUNrQ,2804
-easylink-0.1.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-easylink-0.1.4.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.4.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.4.dist-info/RECORD,,
+easylink-0.1.6.dist-info/METADATA,sha256=Jbmi6F9PaLYIOqs63ViVjqaO6-3StEA7d_PE5N1UYik,2804
+easylink-0.1.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+easylink-0.1.6.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.6.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.6.dist-info/RECORD,,