easylink 0.1.7__tar.gz → 0.1.9__tar.gz

{easylink-0.1.7 → easylink-0.1.9}/CHANGELOG.rst

@@ -1,3 +1,13 @@
+**0.1.9 - 3/14/25**
+
+ - Refactor EmbarrassinglyParallelStep to require a Step during construction
+
+**0.1.8 - 3/13/25**
+
+ - Refactor subgraph logic from Step to HierarchicalStep
+ - Refactor ChoiceStep so that each choice requires a single Step instead of nodes/edges
+ - Standardize the passing around of configurations to be step config instead of parent config
+
 **0.1.7 - 2/26/25**
 
  - Implement initial embarrassingly/auto-parallel step support

{easylink-0.1.7 → easylink-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: easylink
-Version: 0.1.7
+Version: 0.1.9
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.7 → easylink-0.1.9}/pyproject.toml

@@ -29,12 +29,16 @@ exclude = [
     'src/easylink/pipeline_graph.py',
     'src/easylink/pipeline.py',
     'src/easylink/pipeline_schema.py',
+    'src/easylink/pipeline_schema_constants/testing.py',
+    'src/easylink/pipeline_schema_constants/development.py',
     'src/easylink/rule.py',
     'src/easylink/runner.py',
     'src/easylink/step.py',
     'src/easylink/steps/dev/python_pandas/dummy_step.py',
     'src/easylink/steps/dev/python_pyspark/dummy_step.py',
     'src/easylink/steps/dev/test.py',
+    'src/easylink/utilities/aggregator_utils.py',
+    'src/easylink/utilities/splitter_utils.py',
     'src/easylink/utilities/validation_utils.py',
     'tests/integration/test_snakemake_slurm.py',
     'tests/unit/test_graph_components.py',

easylink-0.1.9/src/easylink/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.9"

{easylink-0.1.7 → easylink-0.1.9}/src/easylink/graph_components.py

@@ -259,9 +259,13 @@ class SlotMapping(ABC):
     """A mapping between a slot on a parent node and a slot on one of its child nodes.
 
     ``SlotMapping`` is an interface intended to be used by concrete :class:`InputSlotMapping`
-    and :class:`OutputSlotMapping` classes. It represents a mapping between
-    parent and child nodes at different levels of a potentially-nested
-    :class:`~easylink.pipeline_schema.PipelineSchema`.
+    and :class:`OutputSlotMapping` classes to represent a mapping between parent
+    and child nodes at different levels of a potentially-nested graph. Specifically,
+    they are used to (1) remap edges between parent and child nodes in a
+    :class:`~easylink.pipeline_schema.PipelineSchema` and (2) map a leaf
+    :class:`Step's<easylink.step.Step>` slots to the corresponding
+    :class:`~easylink.implementation.Implementation` slots when building the
+    :class:`~easylink.graph_components.ImplementationGraph`.
 
     Notes
     -----

{easylink-0.1.7 → easylink-0.1.9}/src/easylink/pipeline_schema.py

@@ -22,8 +22,8 @@ class PipelineSchema(HierarchicalStep):
     """All possible pipelines that are fully supported.
 
     A ``PipelineSchema`` is a :class:`~easylink.step.HierarchicalStep` whose
-    :class:`~easylink.graph_components.StepGraph` determines all possible allowable pipelines.
-    The fundamental purpose of this class is to validate that the user-requested
+    :class:`~easylink.graph_components.StepGraph` determines all possible allowable
+    pipelines. The fundamental purpose of this class is to validate that the user-requested
     pipeline to run conforms to a fully supported pipeline.
 
     See :class:`~easylink.step.HierarchicalStep` for inherited attributes.
@@ -80,9 +80,9 @@ class PipelineSchema(HierarchicalStep):
         the user-provided pipeline specification file.
         """
         return super().validate_step(
-            step_config={"substeps": pipeline_config["steps"]},
-            combined_implementations=pipeline_config["combined_implementations"],
-            input_data_config=input_data_config,
+            LayeredConfigTree({"substeps": pipeline_config.steps.to_dict()}),
+            pipeline_config.combined_implementations,
+            input_data_config,
         )
 
     def validate_inputs(self, input_data: dict[str, Path]) -> dict[str, list[str]]:
@@ -133,8 +133,8 @@ class PipelineSchema(HierarchicalStep):
         """
         self._configuration_state = NonLeafConfigurationState(
             self,
-            pipeline_config["steps"],
-            combined_implementations=pipeline_config["combined_implementations"],
+            pipeline_config.steps,
+            combined_implementations=pipeline_config.combined_implementations,
             input_data_config=input_data_config,
         )
 

{easylink-0.1.7 → easylink-0.1.9}/src/easylink/pipeline_schema_constants/__init__.py

@@ -1,3 +1,14 @@
+"""
+=========================
+Pipeline Schema Constants
+=========================
+
+An EasyLink :class:`~easylink.pipeline_schema.PipelineSchema` is a collection of
+:class:`Steps<easylink.step.Step>` that define a fully supported pipeline. This 
+package defines the nodes and edges required to instantiate such ``PipelineSchemas``.
+
+"""
+
 from easylink.pipeline_schema_constants import development, testing
 
 ALLOWED_SCHEMA_PARAMS = {

easylink-0.1.9/src/easylink/pipeline_schema_constants/development.py

@@ -0,0 +1,305 @@
+"""
+=====================================
+Development Pipeline Schema Constants
+=====================================
+
+This module contains the parameters required to instantiate the 
+:class:`~easylink.pipeline_schema.PipelineSchema` for the so-called "development" 
+pipeline, i.e. the pipeline used strictly for development purposes as opposed to
+real entity resolution since it relies on dummy steps, data, and containers.
+
+"""
+
+from easylink.graph_components import (
+    EdgeParams,
+    InputSlot,
+    InputSlotMapping,
+    OutputSlot,
+    OutputSlotMapping,
+)
+from easylink.step import (
+    ChoiceStep,
+    EmbarrassinglyParallelStep,
+    HierarchicalStep,
+    InputStep,
+    LoopStep,
+    OutputStep,
+    ParallelStep,
+    Step,
+)
+from easylink.utilities.aggregator_utils import concatenate_datasets
+from easylink.utilities.splitter_utils import split_data_by_size
+from easylink.utilities.validation_utils import validate_input_file_dummy
+
+NODES = [
+    InputStep(),
+    ParallelStep(
+        template_step=Step(
+            step_name="step_1",
+            input_slots=[
+                InputSlot(
+                    name="step_1_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                ),
+            ],
+            output_slots=[OutputSlot("step_1_main_output")],
+        ),
+    ),
+    Step(
+        step_name="step_2",
+        input_slots=[
+            InputSlot(
+                name="step_2_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            )
+        ],
+        output_slots=[OutputSlot("step_2_main_output")],
+    ),
+    LoopStep(
+        template_step=EmbarrassinglyParallelStep(
+            step=Step(
+                step_name="step_3",
+                input_slots=[
+                    InputSlot(
+                        name="step_3_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                        splitter=split_data_by_size,
+                    ),
+                ],
+                output_slots=[
+                    OutputSlot(
+                        name="step_3_main_output",
+                        aggregator=concatenate_datasets,
+                    ),
+                ],
+            ),
+        ),
+        self_edges=[
+            EdgeParams(
+                source_node="step_3",
+                target_node="step_3",
+                output_slot="step_3_main_output",
+                input_slot="step_3_main_input",
+            )
+        ],
+    ),
+    ChoiceStep(
+        step_name="choice_section",
+        input_slots=[
+            InputSlot(
+                name="choice_section_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            ),
+            InputSlot(
+                name="choice_section_secondary_input",
+                env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            ),
+        ],
+        output_slots=[OutputSlot("choice_section_main_output")],
+        choices={
+            "simple": {
+                "step": HierarchicalStep(
+                    step_name="step_4",
+                    input_slots=[
+                        InputSlot(
+                            name="step_4_main_input",
+                            env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                            validator=validate_input_file_dummy,
+                        ),
+                        InputSlot(
+                            name="step_4_secondary_input",
+                            env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
+                            validator=validate_input_file_dummy,
+                        ),
+                    ],
+                    output_slots=[OutputSlot("step_4_main_output")],
+                    nodes=[
+                        Step(
+                            step_name="step_4a",
+                            input_slots=[
+                                InputSlot(
+                                    name="step_4a_main_input",
+                                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                                InputSlot(
+                                    name="step_4a_secondary_input",
+                                    env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                            ],
+                            output_slots=[OutputSlot("step_4a_main_output")],
+                        ),
+                        Step(
+                            step_name="step_4b",
+                            input_slots=[
+                                InputSlot(
+                                    name="step_4b_main_input",
+                                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                                InputSlot(
+                                    name="step_4b_secondary_input",
+                                    env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                            ],
+                            output_slots=[OutputSlot("step_4b_main_output")],
+                        ),
+                    ],
+                    edges=[
+                        EdgeParams(
+                            source_node="step_4a",
+                            target_node="step_4b",
+                            output_slot="step_4a_main_output",
+                            input_slot="step_4b_main_input",
+                        ),
+                    ],
+                    input_slot_mappings=[
+                        InputSlotMapping(
+                            parent_slot="step_4_main_input",
+                            child_node="step_4a",
+                            child_slot="step_4a_main_input",
+                        ),
+                        InputSlotMapping(
+                            parent_slot="step_4_secondary_input",
+                            child_node="step_4a",
+                            child_slot="step_4a_secondary_input",
+                        ),
+                        InputSlotMapping(
+                            parent_slot="step_4_secondary_input",
+                            child_node="step_4b",
+                            child_slot="step_4b_secondary_input",
+                        ),
+                    ],
+                    output_slot_mappings=[
+                        OutputSlotMapping(
+                            parent_slot="step_4_main_output",
+                            child_node="step_4b",
+                            child_slot="step_4b_main_output",
+                        ),
+                    ],
+                ),
+                "input_slot_mappings": [
+                    InputSlotMapping(
+                        parent_slot="choice_section_main_input",
+                        child_node="step_4",
+                        child_slot="step_4_main_input",
+                    ),
+                    InputSlotMapping(
+                        parent_slot="choice_section_secondary_input",
+                        child_node="step_4",
+                        child_slot="step_4_secondary_input",
+                    ),
+                ],
+                "output_slot_mappings": [
+                    OutputSlotMapping(
+                        parent_slot="choice_section_main_output",
+                        child_node="step_4",
+                        child_slot="step_4_main_output",
+                    ),
+                ],
+            },
+            "complex": {
+                "step": HierarchicalStep(
+                    step_name="step_5_and_6",
+                    nodes=[
+                        Step(
+                            step_name="step_5",
+                            input_slots=[
+                                InputSlot(
+                                    name="step_5_main_input",
+                                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                            ],
+                            output_slots=[OutputSlot("step_5_main_output")],
+                        ),
+                        Step(
+                            step_name="step_6",
+                            input_slots=[
+                                InputSlot(
+                                    name="step_6_main_input",
+                                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                                    validator=validate_input_file_dummy,
+                                ),
+                            ],
+                            output_slots=[OutputSlot("step_6_main_output")],
+                        ),
+                    ],
+                    edges=[
+                        EdgeParams(
+                            source_node="step_5",
+                            target_node="step_6",
+                            output_slot="step_5_main_output",
+                            input_slot="step_6_main_input",
+                        ),
+                    ],
+                ),
+                "input_slot_mappings": [
+                    InputSlotMapping(
+                        parent_slot="choice_section_main_input",
+                        child_node="step_5",
+                        child_slot="step_5_main_input",
+                    ),
+                ],
+                "output_slot_mappings": [
+                    OutputSlotMapping(
+                        parent_slot="choice_section_main_output",
+                        child_node="step_6",
+                        child_slot="step_6_main_output",
+                    ),
+                ],
+            },
+        },
+    ),
+    OutputStep(
+        input_slots=[
+            InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
+        ],
+    ),
+]
+EDGES = [
+    EdgeParams(
+        source_node="input_data",
+        target_node="step_1",
+        output_slot="all",
+        input_slot="step_1_main_input",
+    ),
+    EdgeParams(
+        source_node="step_1",
+        target_node="step_2",
+        output_slot="step_1_main_output",
+        input_slot="step_2_main_input",
+    ),
+    EdgeParams(
+        source_node="step_2",
+        target_node="step_3",
+        output_slot="step_2_main_output",
+        input_slot="step_3_main_input",
+    ),
+    EdgeParams(
+        source_node="step_3",
+        target_node="choice_section",
+        output_slot="step_3_main_output",
+        input_slot="choice_section_main_input",
+    ),
+    EdgeParams(
+        source_node="choice_section",
+        target_node="results",
+        output_slot="choice_section_main_output",
+        input_slot="result",
+    ),
+    EdgeParams(
+        source_node="input_data",
+        target_node="choice_section",
+        output_slot="all",
+        input_slot="choice_section_secondary_input",
+    ),
+]
+SCHEMA_PARAMS = (NODES, EDGES)

{easylink-0.1.7 → easylink-0.1.9}/src/easylink/pipeline_schema_constants/testing.py

@@ -1,7 +1,11 @@
 """
-=====================
-Test Pipeline Schemas
-=====================
+=================================
+Testing Pipeline Schema Constants
+=================================
+
+This module contains the parameters required to instantiate various 
+:class:`~easylink.pipeline_schema.PipelineSchema` used strictly for testing purposes.
+
 """
 
 from easylink.graph_components import (