easylink 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.10"
+__version__ = "0.1.11"

easylink/pipeline_schema.py

@@ -13,7 +13,7 @@ from pathlib import Path
 
 from layered_config_tree import LayeredConfigTree
 
-from easylink.graph_components import EdgeParams
+from easylink.graph_components import EdgeParams, ImplementationGraph
 from easylink.pipeline_schema_constants import ALLOWED_SCHEMA_PARAMS
 from easylink.step import HierarchicalStep, NonLeafConfigurationState, Step
 
@@ -54,6 +54,26 @@ class PipelineSchema(HierarchicalStep):
     def __repr__(self) -> str:
         return f"PipelineSchema.{self.name}"
 
+    def get_implementation_graph(self) -> ImplementationGraph:
+        """Gets the :class:`~easylink.graph_components.ImplementationGraph`.
+
+        The ``PipelineSchema`` is by definition a :class:`~easylink.step.HierarchicalStep`
+        which has a :class:`~easylink.graph_components.StepGraph` containing
+        sub-:class:`Steps<easylink.step.Step>` that need to be unrolled. This method
+        recursively traverses that ``StepGraph`` and its childrens' ``StepGraphs``
+        until all sub-``Steps`` are in a :class:`~easylink.step.LeafConfigurationState`,
+        i.e. all ``Steps`` are implemented by a single ``Implementation`` and we
+        have the desired ``ImplementationGraph``.
+
+        Returns
+        -------
+            The ``ImplementationGraph`` of this ``PipelineSchema``.
+        """
+        implementation_graph = ImplementationGraph()
+        self.add_nodes_to_implementation_graph(implementation_graph)
+        self.add_edges_to_implementation_graph(implementation_graph)
+        return implementation_graph
+
     def validate_step(
         self, pipeline_config: LayeredConfigTree, input_data_config: LayeredConfigTree
     ) -> dict[str, list[str]]:

easylink/step.py

@@ -15,7 +15,6 @@ import copy
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import Any
 
 from layered_config_tree import LayeredConfigTree
 
@@ -250,18 +249,25 @@ class Step:
                 ]
         return errors
 
-    def get_implementation_graph(self) -> ImplementationGraph:
-        """Gets this ``Step's`` :class:`~easylink.graph_components.ImplementationGraph`.
+    def add_nodes_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds the ``Implementations`` related to this ``Step`` as nodes to the :class:`~easylink.graph_components.ImplementationGraph`.
 
-        The ``ImplementationGraph`` and how it is determined depends on whether
-        this ``Step`` is a leaf or a non-leaf, i.e. what its :attr:`configuration_state`
-        is.
+        How the nodes get added depends on whether this ``Step`` is a leaf or a non-leaf,
+        i.e. what its :attr:`configuration_state` is.
+        """
+        self.configuration_state.add_nodes_to_implementation_graph(implementation_graph)
 
-        Returns
-        -------
-            The ``ImplementationGraph`` of this ``Step`` based on its ``configuration_state``.
+    def add_edges_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds the edges of this ``Step's`` ``Implementation(s)`` to the :class:`~easylink.graph_components.ImplementationGraph`.
+
+        How the edges get added depends on whether this ``Step`` is a leaf or a non-leaf,
+        i.e. what its :attr:`configuration_state` is.
         """
-        return self.configuration_state.get_implementation_graph()
+        self.configuration_state.add_edges_to_implementation_graph(implementation_graph)
 
     def get_implementation_edges(self, edge: EdgeParams) -> list[EdgeParams]:
         """Gets the edge information for the ``Implementation`` related to this ``Step``.
@@ -387,7 +393,7 @@ class IOStep(Step):
             The internal configuration of this ``Step``, i.e. it should not include
             the ``Step's`` name.
         combined_implementations
-            The configuration for any implementations to be combined.
+            The configuration for any ``Implementations`` to be combined.
         input_data_config
             The input data configuration for the entire pipeline.
         """
@@ -395,8 +401,10 @@ class IOStep(Step):
             self, step_config, combined_implementations, input_data_config
         )
 
-    def get_implementation_graph(self) -> ImplementationGraph:
-        """Gets this ``Step's`` :class:`~easylink.graph_components.ImplementationGraph`.
+    def add_nodes_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds this ``IOStep's`` ``Implementation`` as a node to the :class:`~easylink.graph_components.ImplementationGraph`.
 
         Notes
         -----
@@ -404,19 +412,22 @@ class IOStep(Step):
         via an :class:`~easylink.implementation.Implementation`. As such, we
         leverage the :class:`~easylink.implementation.NullImplementation` class
         to generate the graph node.
-
-        Returns
-        -------
-            The ``ImplementationGraph`` of this ``Step``.
         """
-        implementation_graph = ImplementationGraph()
         implementation_graph.add_node_from_implementation(
             self.name,
             implementation=NullImplementation(
                 self.name, self.input_slots.values(), self.output_slots.values()
             ),
         )
-        return implementation_graph
+
+    def add_edges_to_implementation_graph(self, implementation_graph):
+        """Adds the edges of this ``Step's`` ``Implementation`` to the ``ImplementationGraph``.
+
+        ``IOSteps`` do not have edges within them in the ``ImplementationGraph``,
+        since they are represented by a single ``NullImplementation`` node, and so we
+        simply pass.
+        """
+        pass
 
 
 class InputStep(IOStep):
@@ -1394,8 +1405,17 @@ class ConfigurationState(ABC):
         """The input data configuration for the entire pipeline."""
 
     @abstractmethod
-    def get_implementation_graph(self) -> ImplementationGraph:
-        """Resolves the graph composed of ``Steps`` into one composed of ``Implementations``."""
+    def add_nodes_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds this ``Step's`` ``Implementation(s)`` as nodes to the :class:`~easylink.graph_components.ImplementationGraph`."""
+        pass
+
+    @abstractmethod
+    def add_edges_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds the edges of this ``Step's`` ``Implementation(s)`` to the :class:`~easylink.graph_components.ImplementationGraph`."""
         pass
 
     @abstractmethod
@@ -1438,19 +1458,16 @@ class LeafConfigurationState(ConfigurationState):
             else self.step_config.implementation
         )
 
-    def get_implementation_graph(self) -> ImplementationGraph:
-        """Gets this ``Step's`` :class:`~easylink.graph_components.ImplementationGraph`.
+    def add_nodes_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds this ``Step's`` :class:`~easylink.implementation.Implementation` as a node to the :class:`~easylink.graph_components.ImplementationGraph`.
 
         A ``Step`` in a leaf configuration state by definition has no sub-``Steps``
         to unravel; we are able to directly instantiate an :class:`~easylink.implementation.Implementation`
-        and generate an ``ImplementationGraph`` from it.
-
-        Returns
-        -------
-            The ``ImplementationGraph`` related to this ``Step``.
+        and add it to the ``ImplementationGraph``.
         """
         step = self._step
-        implementation_graph = ImplementationGraph()
         if self.is_combined:
             if isinstance(step, EmbarrassinglyParallelStep):
                 raise NotImplementedError(
@@ -1475,7 +1492,15 @@ class LeafConfigurationState(ConfigurationState):
             step.implementation_node_name,
             implementation=implementation,
         )
-        return implementation_graph
+
+    def add_edges_to_implementation_graph(self, implementation_graph) -> None:
+        """Adds the edges for this ``Step's`` ``Implementation`` to the ``ImplementationGraph``.
+
+        ``Steps`` in a ``LeafConfigurationState`` do not actually have edges within them
+        (they are represented by a single node in the ``ImplementationGraph``) and so
+        we simply pass.
+        """
+        pass
 
     def get_implementation_edges(self, edge: EdgeParams) -> list[EdgeParams]:
         """Gets the edge information for the ``Implementation`` related to this ``Step``.
@@ -1543,7 +1568,8 @@ class NonLeafConfigurationState(ConfigurationState):
         for; it should not include the ``Step's`` name (though it must include
         the sub-step names).
     combined_implementations
-        The configuration for any implementations to be combined.
+        The configuration for any :class:`Implementations<easylink.implementation.Implementation>`
+        to be combined.
     input_data_config
         The input data configuration for the entire pipeline.
 
@@ -1585,61 +1611,62 @@ class NonLeafConfigurationState(ConfigurationState):
         self._nodes = step.step_graph.nodes
         self._configure_subgraph_steps()
 
-    def get_implementation_graph(self) -> ImplementationGraph:
-        """Gets this ``Step's`` :class:`~easylink.graph_components.ImplementationGraph`.
+    def add_nodes_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds this ``Step's`` ``Implementations`` as nodes to the ``ImplementationGraph``.
 
-        A ``Step`` in a non-leaf configuration state by definition has a ``StepGraph``
-        containing sub-``Steps`` that  need to be unrolled. This method recursively
-        traverses that ``StepGraph`` and its childrens' ``StepGraphs`` until all
-        sub-``Steps`` are in a :class:`LeafConfigurationState`, i.e. all ``Steps``
-        are implemented by a single ``Implementation`` and we have our desired
-        ``ImplementationGraph``.
+        This is a recursive function; it calls itself until all sub-``Steps``
+        are of a ``LeafConfigurationState`` and have had their corresponding
+        ``Implementations`` added as nodes to the ``ImplementationGraph``.
+        """
+        for node in self._step.step_graph.nodes:
+            substep = self._step.step_graph.nodes[node]["step"]
+            substep.add_nodes_to_implementation_graph(implementation_graph)
 
-        Returns
-        -------
-            The ``ImplementationGraph`` of this ``Step``.
+    def add_edges_to_implementation_graph(
+        self, implementation_graph: ImplementationGraph
+    ) -> None:
+        """Adds the edges of this ``Step's`` ``Implementations`` to the ``ImplementationGraph``.
 
-        Notes
-        -----
-        This method is first called on the entire :class:`~easylink.pipeline_schema.PipelineSchema`
-        when constructing the :class:`~easylink.pipeline_graph.PipelineGraph`
-        to run.
+        This method does two things:
+        1. Adds the edges *at this level* (i.e. at the ``Step`` tied to this
+        ``NonLeafConfigurationState``) to the ``ImplementationGraph``.
+        2. Recursively traverses all sub-steps and adds their edges to the
+        ``ImplementationGraph``.
 
+        Note that to achieve (1), edges must be mapped from being between steps at
+        this level of the hierarchy, all the way down to being between concrete implementations.
+        Mapping each edge down to the implementation level is *itself* a recursive
+        operation (see ``get_implementation_edges``).
         """
-        implementation_graph = ImplementationGraph()
-        self.add_nodes(implementation_graph)
-        self.add_edges(implementation_graph)
-        return implementation_graph
-
-    def add_nodes(self, implementation_graph: ImplementationGraph) -> None:
-        """Adds nodes for each ``Step`` to the ``ImplementationGraph``."""
-        for node in self._nodes:
-            step = self._nodes[node]["step"]
-            implementation_graph.update(step.get_implementation_graph())
-
-    def add_edges(self, implementation_graph: ImplementationGraph) -> None:
-        """Adds the edges to the ``ImplementationGraph``."""
+        # Add the edges at this level (i.e. the edges at this `self._step`)
         for source, target, edge_attrs in self._step.step_graph.edges(data=True):
-            all_edges = []
             edge = EdgeParams.from_graph_edge(source, target, edge_attrs)
-            parent_source_step = self._nodes[source]["step"]
-            parent_target_step = self._nodes[target]["step"]
+            source_step = self._nodes[source]["step"]
+            target_step = self._nodes[target]["step"]
 
-            source_edges = parent_source_step.get_implementation_edges(edge)
+            source_edges = source_step.get_implementation_edges(edge)
             for source_edge in source_edges:
-                for target_edge in parent_target_step.get_implementation_edges(source_edge):
-                    all_edges.append(target_edge)
+                for target_edge in target_step.get_implementation_edges(source_edge):
+                    implementation_graph.add_edge_from_params(target_edge)
 
-            for edge in all_edges:
-                implementation_graph.add_edge_from_params(edge)
+        # Recurse through all sub-steps and add the edges between them
+        for node in self._step.step_graph.nodes:
+            substep = self._step.step_graph.nodes[node]["step"]
+            substep.add_edges_to_implementation_graph(implementation_graph)
 
     def get_implementation_edges(self, edge: EdgeParams) -> list[EdgeParams]:
-        """Gets the edge information for the ``Implementation`` related to this ``Step``.
+        """Gets the edges for the ``Implementation`` related to this ``Step``.
+
+        This method maps an edge between ``Steps`` in this ``Step's`` ``StepGraph``
+        to one or more edges between ``Implementations`` by applying ``SlotMappings``.
 
         Parameters
         ----------
         edge
-            The ``Step's`` edge information to be propagated to the ``ImplementationGraph``.
+            The edge information of the edge in the ``StepGraph`` to be mapped to
+            the ``Implementation`` level.
 
         Raises
         ------
@@ -1648,7 +1675,28 @@ class NonLeafConfigurationState(ConfigurationState):
 
         Returns
         -------
-            The ``Implementation's`` edge information.
+            A list of edges between ``Implementations`` which are ready to add to
+            the ``ImplementationGraph``.
+
+        Notes
+        -----
+        In EasyLink, an edge (in either a ``StepGraph`` or ``ImplementationGraph``)
+        sconnects two ``Slot``.
+
+        The core of this method is to map the ``Slots`` on the ``StepGraph`` edge
+        to the corresponding ``Slots`` on ``Implementations``.
+
+        At each level in the step hierarchy, ``SlotMappings`` indicate how to map
+        a ``Slot`` to the level below in the hierarchy.
+
+        This method recurses through the step hierarchy until it reaches the leaf
+        ``Steps`` relevant to this edge in order to compose all the ``SlotMappings``
+        that should apply to it.
+
+        Because a single ``Step`` can become multiple nodes in the ``ImplementationGraph``
+        (e.g. a :class:`TemplatedStep`), a single edge between ``Steps`` may actually
+        become multiple edges between ``Implementations``, which is why this method
+        can return a list.
         """
         implementation_edges = []
         if edge.source_node == self._step.name:

{easylink-0.1.10.dist-info → easylink-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.10
+Version: 0.1.11
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.10.dist-info → easylink-0.1.11.dist-info}/RECORD

@@ -1,6 +1,6 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=z0zCHFTcKSR0tJ6h5qrpNmRVP21QIPP8N0p7quCnnm0,23
+easylink/_version.py,sha256=nllDrH0jyChMuuYrK0CC55iTBKUNTUjejtcwxyUF2EQ,23
 easylink/cli.py,sha256=ARSKAljepNOEYd1VCS_QqBJQIBLzE3IgKiOb5-OROdY,6380
 easylink/configuration.py,sha256=Ire2pMZNZ6wtSwhcWnQpYa-snX4KrhXgovlQwQ2Wxf4,12530
 easylink/graph_components.py,sha256=PhMKxpgZjorhubS7vcta1pgXgXSGplmPulQpV0YZhqo,14811
@@ -8,10 +8,10 @@ easylink/implementation.py,sha256=AwGl5YCKCSQo91owWj-gg9_5lBz7H_4q2z7jF0BhXs4,89
 easylink/implementation_metadata.yaml,sha256=VvlEu3Dvlmeh1MpzeYx91j22GiV-9mu3hZP5yVuW04o,6763
 easylink/pipeline.py,sha256=EyCXv5p9WzTqcndXK6ukBJE6jY_fWIP_DGZQUl1wRcY,12284
 easylink/pipeline_graph.py,sha256=vsY6nW_iEwZCNf_N_3CsixsKBUy_5JxGEi61-1Q-KAw,22842
-easylink/pipeline_schema.py,sha256=kINpvy2Fl2S3FBqgdgZCCFHEk237_36X4ltLOtk5-dE,5862
+easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
 easylink/rule.py,sha256=W97LMI-vkEPipJbnSZLn2BxfYfFtvzGTKzq6YgDVri0,19913
 easylink/runner.py,sha256=k9ICTToHj2xr6MGIuvlWf6YMeZ47UGgseaMByMgUGac,6271
-easylink/step.py,sha256=8EhoFOXBLWgDfb3OhmQu5g03fqElIJCWg8-Y_5azKEA,67100
+easylink/step.py,sha256=ttteoyIiwnlDjc6QxWKO85GI9ubVGu9Oy5XDG2u4YrY,69885
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
 easylink/pipeline_schema_constants/__init__.py,sha256=uRVjQw7_Ff5IBQw0_Jc93Fzfa-MnbPVPKsy18CCaW7E,1021
@@ -43,8 +43,8 @@ easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,9
 easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
 easylink/utilities/splitter_utils.py,sha256=y4CbbTBgRaoXFxy-9Eu5eWx4lA4ZEcbrYpxgLIzG_kc,2602
 easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
-easylink-0.1.10.dist-info/METADATA,sha256=wPKznMaCPytiFoECbqY0jcFra-Hl28FMQJLo4XQcwdo,2805
-easylink-0.1.10.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-easylink-0.1.10.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.10.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.10.dist-info/RECORD,,
+easylink-0.1.11.dist-info/METADATA,sha256=LCqBkp3ndZAQF8Dwo-4ugO2yLyyduY7oRwg8EcBX0lY,2805
+easylink-0.1.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+easylink-0.1.11.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.11.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.11.dist-info/RECORD,,

{easylink-0.1.10.dist-info → easylink-0.1.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any