easylink 0.1.11__py3-none-any.whl → 0.1.13__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.11"
+__version__ = "0.1.13"

easylink/pipeline_schema_constants/development.py

@@ -66,16 +66,37 @@ NODES = [
                         name="step_3_main_input",
                         env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
                         validator=validate_input_file_dummy,
-                        splitter=split_data_by_size,
                     ),
                 ],
                 output_slots=[
                     OutputSlot(
                         name="step_3_main_output",
-                        aggregator=concatenate_datasets,
                     ),
                 ],
             ),
+            input_slots=[
+                InputSlot(
+                    name="step_3_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                    splitter=split_data_by_size,
+                ),
+            ],
+            output_slots=[OutputSlot("step_3_main_output", aggregator=concatenate_datasets)],
+            input_slot_mappings=[
+                InputSlotMapping(
+                    parent_slot="step_3_main_input",
+                    child_node="step_3",
+                    child_slot="step_3_main_input",
+                ),
+            ],
+            output_slot_mappings=[
+                OutputSlotMapping(
+                    parent_slot="step_3_main_output",
+                    child_node="step_3",
+                    child_slot="step_3_main_output",
+                ),
+            ],
         ),
         self_edges=[
             EdgeParams(

easylink/step.py

@@ -87,6 +87,7 @@ class Step:
         output_slots: Iterable[OutputSlot] = (),
         input_slot_mappings: Iterable[InputSlotMapping] = (),
         output_slot_mappings: Iterable[OutputSlotMapping] = (),
+        is_embarrassingly_parallel: bool = False,
     ) -> None:
         self.step_name = step_name
         """The name of the pipeline step in the ``PipelineSchema``. It must also match
@@ -107,6 +108,8 @@ class Step:
         }
         """A combined dictionary containing both the ``InputSlotMappings`` and
         ``OutputSlotMappings`` of this ``Step``."""
+        self.is_embarrassingly_parallel = is_embarrassingly_parallel
+        """Whether or not this ``Step`` is to be run in an embarrassingly parallel manner."""
         self.parent_step = None
         """This ``Step's`` parent ``Step``, if applicable."""
         self._configuration_state = None
@@ -884,7 +887,7 @@ class TemplatedStep(Step, ABC):
             self.step_graph = StepGraph()
             self.template_step.name = self.name
             self.step_graph.add_node_from_step(self.template_step)
-            # Special handle the slot_mappings update
+            # Update the slot mappings with renamed children
             input_mappings = [
                 InputSlotMapping(slot, self.name, slot) for slot in self.input_slots
             ]
@@ -899,10 +902,8 @@ class TemplatedStep(Step, ABC):
             num_repeats = len(expanded_config)
             self.step_graph = self._update_step_graph(num_repeats)
             self.slot_mappings = self._update_slot_mappings(num_repeats)
-        # Manually set the configuration state to non-leaf instead of relying
-        # on super().get_configuration_state() because that method will erroneously
-        # set to leaf state in the event the user didn't include the config_key
-        # in the pipeline specification.
+
+        # TemplatedSteps are by definition non-leaf steps.
         self._configuration_state = NonLeafConfigurationState(
             self, expanded_config, combined_implementations, input_data_config
         )
@@ -1164,10 +1165,22 @@ class EmbarrassinglyParallelStep(Step):
     def __init__(
         self,
         step: Step,
+        input_slots: Iterable[InputSlot],
+        output_slots: Iterable[OutputSlot],
+        input_slot_mappings: Iterable[InputSlotMapping],
+        output_slot_mappings: Iterable[OutputSlotMapping],
     ) -> None:
         super().__init__(
-            step.step_name, step.name, step.input_slots.values(), step.output_slots.values()
+            step.step_name,
+            step.name,
+            input_slots,
+            output_slots,
+            input_slot_mappings,
+            output_slot_mappings,
+            is_embarrassingly_parallel=True,
         )
+        self.step_graph = None
+        self.step = step
         self._validate()
 
     def _validate(self) -> None:
@@ -1210,6 +1223,50 @@ class EmbarrassinglyParallelStep(Step):
         if errors:
             raise ValueError("\n".join(errors))
 
+    def set_configuration_state(
+        self,
+        step_config: LayeredConfigTree,
+        combined_implementations: LayeredConfigTree,
+        input_data_config: LayeredConfigTree,
+    ):
+        """Sets the configuration state to 'non-leaf'.
+
+        In addition to setting the configuration state, this also updates the
+        :class:`~easylink.graph_components.StepGraph` and
+        :class:`SlotMappings<easylink.graph_components.SlotMapping>`.
+
+        Parameters
+        ----------
+        step_config
+            The internal configuration of this ``Step``, i.e. it should not include
+            the ``Step's`` name.
+        combined_implementations
+            The configuration for any implementations to be combined.
+        input_data_config
+            The input data configuration for the entire pipeline.
+        """
+        if self.step.name != self.name:
+            # Update the step name if the parent got renamed, e.g. a parent LoopStep
+            # 'step_1' that got expanded to 'step_1_loop_1', etc.
+            self.step.name = self.name
+            input_mappings = [
+                InputSlotMapping(slot, self.name, slot) for slot in self.input_slots
+            ]
+            output_mappings = [
+                OutputSlotMapping(slot, self.name, slot) for slot in self.output_slots
+            ]
+            self.slot_mappings = {"input": input_mappings, "output": output_mappings}
+        # Generate step graph from the single ``step`` attr
+        self.step_graph = StepGraph()
+        self.step_graph.add_node_from_step(self.step)
+        # Add the key back to the expanded config
+        expanded_config = LayeredConfigTree({self.name: step_config})
+
+        # EmbarrassinglyParallelSteps are by definition non-leaf steps
+        self._configuration_state = NonLeafConfigurationState(
+            self, expanded_config, combined_implementations, input_data_config
+        )
+
 
 class ChoiceStep(Step):
     """A type of :class:`Step` that allows for choosing from a set of options.
@@ -1469,10 +1526,10 @@ class LeafConfigurationState(ConfigurationState):
         """
         step = self._step
         if self.is_combined:
-            if isinstance(step, EmbarrassinglyParallelStep):
+            if step.is_embarrassingly_parallel:
                 raise NotImplementedError(
                     "Combining implementations with embarrassingly parallel steps "
-                    "is not yet supported."
+                    "is not supported."
                 )
             implementation = PartialImplementation(
                 combined_name=self.step_config[COMBINED_IMPLEMENTATION_KEY],
@@ -1486,7 +1543,7 @@ class LeafConfigurationState(ConfigurationState):
                 implementation_config=self.implementation_config,
                 input_slots=step.input_slots.values(),
                 output_slots=step.output_slots.values(),
-                is_embarrassingly_parallel=isinstance(step, EmbarrassinglyParallelStep),
+                is_embarrassingly_parallel=step.is_embarrassingly_parallel,
             )
         implementation_graph.add_node_from_implementation(
             step.implementation_node_name,
@@ -1608,7 +1665,6 @@ class NonLeafConfigurationState(ConfigurationState):
                 "NonLeafConfigurationState requires a subgraph upon which to operate, "
                 f"but Step {step.name} has no step graph."
             )
-        self._nodes = step.step_graph.nodes
         self._configure_subgraph_steps()
 
     def add_nodes_to_implementation_graph(
@@ -1622,8 +1678,59 @@ class NonLeafConfigurationState(ConfigurationState):
         """
         for node in self._step.step_graph.nodes:
             substep = self._step.step_graph.nodes[node]["step"]
+            if self._step.is_embarrassingly_parallel:
+                substep.is_embarrassingly_parallel = True
+                self._propagate_splitter_aggregators(self._step, substep)
             substep.add_nodes_to_implementation_graph(implementation_graph)
 
+    @staticmethod
+    def _propagate_splitter_aggregators(parent: Step, child: Step):
+        """Propagates splitters and aggregators to child ``Steps``.
+
+        This method adds the :meth:`~easylink.graph_components.InputSlot.splitter`
+        and :meth:`~easylink.graph_components.OutputSlot.aggregator` methods from a
+        parent ``Step's`` :class:`~easylink.graph_components.InputSlot` and
+        :class:`OutputSlots<easylink.graph_components.OutputSlot>` to the corresponding
+        child steps' slots.
+
+        Parameters
+        ----------
+        parent
+            The parent ``Step`` whose ``splitter`` and ``aggregator`` methods are
+            to be propagated to the appropriate child ``Step``.
+        child
+            A child ``Step`` to potentially have its parent's ``splitter`` and
+            ``aggregators`` assigned to its ``InputSlot`` and ``OutputSlots``,
+            respectively.
+        """
+        for parent_input_slot_name, parent_input_slot in parent.input_slots.items():
+            if parent_input_slot.splitter:
+                # Extract the appropriate child slot name from the mapping
+                mappings_with_splitter = [
+                    mapping
+                    for mapping in parent.slot_mappings["input"]
+                    if mapping.parent_slot == parent_input_slot_name
+                ]
+                for mapping in mappings_with_splitter:
+                    child_node = mapping.child_node
+                    child_slot = mapping.child_slot
+                    # Assign the splitter to the appropriate child slot
+                    if child_slot in child.input_slots and child_node == child.name:
+                        child.input_slots[child_slot].splitter = parent_input_slot.splitter
+        for parent_output_slot_name, parent_output_slot in parent.output_slots.items():
+            # Extract the appropriate child slot name from the mapping
+            mappings_from_parent = [
+                mapping
+                for mapping in parent.slot_mappings["output"]
+                if mapping.parent_slot == parent_output_slot_name
+            ]
+            for mapping in mappings_from_parent:
+                child_node = mapping.child_node
+                child_slot = mapping.child_slot
+                # Assign the aggregator to the appropriate child slot
+                if child_slot in child.output_slots and child_node == child.name:
+                    child.output_slots[child_slot].aggregator = parent_output_slot.aggregator
+
     def add_edges_to_implementation_graph(
         self, implementation_graph: ImplementationGraph
     ) -> None:
@@ -1643,8 +1750,8 @@ class NonLeafConfigurationState(ConfigurationState):
         # Add the edges at this level (i.e. the edges at this `self._step`)
         for source, target, edge_attrs in self._step.step_graph.edges(data=True):
             edge = EdgeParams.from_graph_edge(source, target, edge_attrs)
-            source_step = self._nodes[source]["step"]
-            target_step = self._nodes[target]["step"]
+            source_step = self._step.step_graph.nodes[source]["step"]
+            target_step = self._step.step_graph.nodes[target]["step"]
 
             source_edges = source_step.get_implementation_edges(edge)
             for source_edge in source_edges:
@@ -1707,7 +1814,7 @@ class NonLeafConfigurationState(ConfigurationState):
             ]
             for mapping in mappings:
                 new_edge = mapping.remap_edge(edge)
-                new_step = self._nodes[mapping.child_node]["step"]
+                new_step = self._step.step_graph.nodes[mapping.child_node]["step"]
                 imp_edges = new_step.get_implementation_edges(new_edge)
                 implementation_edges.extend(imp_edges)
         elif edge.target_node == self._step.name:
@@ -1718,7 +1825,7 @@ class NonLeafConfigurationState(ConfigurationState):
             ]
             for mapping in mappings:
                 new_edge = mapping.remap_edge(edge)
-                new_step = self._nodes[mapping.child_node]["step"]
+                new_step = self._step.step_graph.nodes[mapping.child_node]["step"]
                 imp_edges = new_step.get_implementation_edges(new_edge)
                 implementation_edges.extend(imp_edges)
         else:
@@ -1733,12 +1840,14 @@ class NonLeafConfigurationState(ConfigurationState):
         This method recursively traverses the ``StepGraph`` and sets the configuration
         state for each ``Step`` until reaching all leaf nodes.
         """
-        for node in self._nodes:
-            step = self._nodes[node]["step"]
+        for sub_node in self._step.step_graph.nodes:
+            sub_step = self._step.step_graph.nodes[sub_node]["step"]
             # IOStep names never appear in configuration
             step_config = (
-                self.step_config if isinstance(step, IOStep) else self.step_config[step.name]
+                self.step_config
+                if isinstance(sub_step, IOStep)
+                else self.step_config[sub_step.name]
             )
-            step.set_configuration_state(
+            sub_step.set_configuration_state(
                 step_config, self.combined_implementations, self.input_data_config
             )

{easylink-0.1.11.dist-info → easylink-0.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.11
+Version: 0.1.13
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.11.dist-info → easylink-0.1.13.dist-info}/RECORD

@@ -1,6 +1,6 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=nllDrH0jyChMuuYrK0CC55iTBKUNTUjejtcwxyUF2EQ,23
+easylink/_version.py,sha256=khDKUuWafURKVs5EAZkpOMiUHI2-V7axlqrWLPUpuZo,23
 easylink/cli.py,sha256=ARSKAljepNOEYd1VCS_QqBJQIBLzE3IgKiOb5-OROdY,6380
 easylink/configuration.py,sha256=Ire2pMZNZ6wtSwhcWnQpYa-snX4KrhXgovlQwQ2Wxf4,12530
 easylink/graph_components.py,sha256=PhMKxpgZjorhubS7vcta1pgXgXSGplmPulQpV0YZhqo,14811
@@ -11,11 +11,11 @@ easylink/pipeline_graph.py,sha256=vsY6nW_iEwZCNf_N_3CsixsKBUy_5JxGEi61-1Q-KAw,22
 easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
 easylink/rule.py,sha256=W97LMI-vkEPipJbnSZLn2BxfYfFtvzGTKzq6YgDVri0,19913
 easylink/runner.py,sha256=k9ICTToHj2xr6MGIuvlWf6YMeZ47UGgseaMByMgUGac,6271
-easylink/step.py,sha256=ttteoyIiwnlDjc6QxWKO85GI9ubVGu9Oy5XDG2u4YrY,69885
+easylink/step.py,sha256=Hweg1OAGcmrNAt95C-M4ksOAtc_db0oeibbF3cnqhq0,74951
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
 easylink/pipeline_schema_constants/__init__.py,sha256=uRVjQw7_Ff5IBQw0_Jc93Fzfa-MnbPVPKsy18CCaW7E,1021
-easylink/pipeline_schema_constants/development.py,sha256=kOTEqfZD5pWqP9gu7E6r9Cubf3ILtWEUxCfJfrN8znc,11547
+easylink/pipeline_schema_constants/development.py,sha256=0fc6xWRBr5e_xDaldR9sY2vMQJU1wnlhDQS_-yUOT6g,12339
 easylink/pipeline_schema_constants/testing.py,sha256=ohcTlT_viZYxS1GkO46mjkb8IzXo6yIOqvBbb4YrOhA,10897
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
@@ -43,8 +43,8 @@ easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,9
 easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
 easylink/utilities/splitter_utils.py,sha256=y4CbbTBgRaoXFxy-9Eu5eWx4lA4ZEcbrYpxgLIzG_kc,2602
 easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
-easylink-0.1.11.dist-info/METADATA,sha256=LCqBkp3ndZAQF8Dwo-4ugO2yLyyduY7oRwg8EcBX0lY,2805
-easylink-0.1.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-easylink-0.1.11.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.11.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.11.dist-info/RECORD,,
+easylink-0.1.13.dist-info/METADATA,sha256=ooL68LseA1cN5X2wLOB_uIFgXIjW0PCINie7aMHw6t0,2805
+easylink-0.1.13.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+easylink-0.1.13.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.13.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.13.dist-info/RECORD,,

{easylink-0.1.11.dist-info → easylink-0.1.13.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any