easylink 0.1.8__tar.gz → 0.1.10__tar.gz

{easylink-0.1.8 → easylink-0.1.10}/CHANGELOG.rst

@@ -1,3 +1,11 @@
+**0.1.10 - 3/25/25**
+
+ - Make InputSlots and OutputSlots mutable
+
+**0.1.9 - 3/14/25**
+
+ - Refactor EmbarrassinglyParallelStep to require a Step during construction
+
 **0.1.8 - 3/13/25**
 
  - Refactor subgraph logic from Step to HierarchicalStep

{easylink-0.1.8/src/easylink.egg-info → easylink-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.8
+Version: 0.1.10
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.8 → easylink-0.1.10}/docs/source/concepts/index.rst

@@ -5,6 +5,12 @@ Concepts
 ========
 Here we cover several core conceptual topics related using EasyLink.
 
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    */index
+
 .. todo::
 
-   Everything
+   Almost everything

easylink-0.1.10/docs/source/concepts/pipeline_schema/index.rst

@@ -0,0 +1,47 @@
+.. _pipeline_schema:
+
+Pipeline Schema
+===============
+
+Motivation
+----------
+
+Imagine the Census Bureau has an entity resolution pipeline that links people between datasets.
+One step in this pipeline, called *blocking*, rules out comparing certain records with each other
+in order to focus only on the pairs of records that might really be links.
+The current pipeline uses a simple blocking mechanism,
+which won't compare two records unless they match exactly on any of a few key attributes.
+Census wants to explore whether using more sophisticated blocking methods would improve results,
+without changing anything else in the pipeline.
+
+Currently, software for entity resolution is mostly created by researchers.
+Each researcher uses the technologies familiar to them and frames the entity resolution task
+in the way that is most natural for their own examples,
+making it hard to use multiple software modules together.
+As a result, trying a new blocking method is too expensive for the Census Bureau
+to undertake without knowing what the benefit will be.
+
+Introduction
+------------
+
+**EasyLink** is a tool for creating entity resolution pipelines
+by chaining together existing pieces of software.
+
+It doesn't allow making pipelines arbitrarily by chaining together whatever software you want however you want,
+and this is actually the key value proposition of EasyLink.
+To be used in pipelines created with EasyLink, software modules must follow standard patterns.
+These standards (including standard data formats) allow a single piece of software
+to be used for the same conceptual task in any entity resolution pipeline.
+
+We define our standards via the *pipeline schema*, which is described on this page.
+The design goals of the pipeline schema are to be:
+
+- **Flexible** enough to capture current entity resolution methods and new methods that are yet to be developed.
+  In particular, we want to avoid a small innovation in one part of a pipeline causing that entire pipeline
+  to become impossible to construct with EasyLink—*"bend, don't break."*
+- **Detailed/standardized** enough in capturing current entity resolution methods,
+  and areas of active methodological research, to allow very fine-grained experiments and interoperability.
+
+Our pipeline schema can also be viewed as a restricted (but still very large) space of possible pipelines.
+That is, there are certain pipelines EasyLink does not allow because they do not conform to our standards,
+and the pipeline schema tells EasyLink how to check whether a pipeline is or isn't allowed.

{easylink-0.1.8 → easylink-0.1.10}/pyproject.toml

@@ -29,6 +29,8 @@ exclude = [
     'src/easylink/pipeline_graph.py',
     'src/easylink/pipeline.py',
     'src/easylink/pipeline_schema.py',
+    'src/easylink/pipeline_schema_constants/testing.py',
+    'src/easylink/pipeline_schema_constants/development.py',
     'src/easylink/rule.py',
     'src/easylink/runner.py',
     'src/easylink/step.py',

easylink-0.1.10/src/easylink/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.10"

{easylink-0.1.8 → easylink-0.1.10}/src/easylink/graph_components.py

@@ -12,7 +12,8 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from collections.abc import Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from types import NotImplementedType
 from typing import TYPE_CHECKING, Any
 
 import networkx as nx
@@ -22,7 +23,7 @@ if TYPE_CHECKING:
     from easylink.step import Step
 
 
-@dataclass(frozen=True)
+@dataclass()
 class InputSlot:
     """A single input slot to a specific node.
 
@@ -41,20 +42,48 @@ class InputSlot:
     env_var: str | None
     """The environment variable that is used to pass a list  of data filepaths to 
     an ``Implementation``."""
-    validator: Callable[[str], None]
+    validator: Callable[[str], None] = field(compare=False)
     """A function that validates the input data being passed into the pipeline via 
     this ``InputSlot``. If the data is invalid, the function should raise an exception 
     with a descriptive error message which will then be reported to the user.
     **Note that the function *must* be defined in the** :mod:`easylink.utilities.validation_utils` 
     **module!**"""
-    splitter: Callable[[list[str], str, Any], None] | None = None
+    splitter: Callable[[list[str], str, Any], None] | None = field(
+        default=None, compare=False
+    )
     """A function that splits the incoming data to this ``InputSlot`` into smaller
     pieces. The primary purpose of this functionality is to run sections of the 
     pipeline in an embarrassingly parallel manner. **Note that the function *must* 
     be defined in the **:mod:`easylink.utilities.splitter_utils`** module!**"""
 
+    def __eq__(self, other: Any) -> bool | NotImplementedType:
+        """Checks if two ``InputSlots`` are equal.
 
-@dataclass(frozen=True)
+        Two ``InputSlots`` are considered equal if their names, ``env_vars``, and
+        names of their ``validators`` and ``splitters`` are all the same.
+        """
+        if not isinstance(other, InputSlot):
+            return NotImplemented
+        splitter_name = self.splitter.__name__ if self.splitter else None
+        other_splitter_name = other.splitter.__name__ if other.splitter else None
+        return (
+            self.name == other.name
+            and self.env_var == other.env_var
+            and self.validator.__name__ == other.validator.__name__
+            and splitter_name == other_splitter_name
+        )
+
+    def __hash__(self) -> int:
+        """Hashes an ``InputSlot``.
+
+        The hash is based on the name of the ``InputSlot``, its ``env_var``, and
+        the names of its ``validator`` and ``splitter``.
+        """
+        splitter_name = self.splitter.__name__ if self.splitter else None
+        return hash((self.name, self.env_var, self.validator.__name__, splitter_name))
+
+
+@dataclass()
 class OutputSlot:
     """A single output slot from a specific node.
 
@@ -75,12 +104,32 @@ class OutputSlot:
 
     name: str
     """The name of the ``OutputSlot``."""
-    aggregator: Callable[[list[str], str], None] = None
+    aggregator: Callable[[list[str], str], None] = field(default=None, compare=False)
     """A function that aggregates all of the generated data to be passed out via this
     ``OutputSlot``. The primary purpose of this functionality is to run sections
     of the pipeline in an embarrassingly parallel manner. **Note that the function 
     *must* be defined in the **:py:mod:`easylink.utilities.aggregator_utils`** module!**"""
 
+    def __eq__(self, other: Any) -> bool | NotImplementedType:
+        """Checks if two ``OutputSlots`` are equal.
+
+        Two ``OutputSlots`` are considered equal if their names and the names of their
+        ``aggregators`` are the same.
+        """
+        if not isinstance(other, OutputSlot):
+            return NotImplemented
+        aggregator_name = self.aggregator.__name__ if self.aggregator else None
+        other_aggregator_name = other.aggregator.__name__ if other.aggregator else None
+        return self.name == other.name and aggregator_name == other_aggregator_name
+
+    def __hash__(self) -> int:
+        """Hashes an ``OutputSlot``.
+
+        The hash is based on the name of the ``OutputSlot`` and the name of its ``aggregator``.
+        """
+        aggregator_name = self.aggregator.__name__ if self.aggregator else None
+        return hash((self.name, aggregator_name))
+
 
 @dataclass(frozen=True)
 class EdgeParams:

{easylink-0.1.8 → easylink-0.1.10}/src/easylink/pipeline_schema_constants/development.py

@@ -59,21 +59,23 @@ NODES = [
     ),
     LoopStep(
         template_step=EmbarrassinglyParallelStep(
-            step_name="step_3",
-            input_slots=[
-                InputSlot(
-                    name="step_3_main_input",
-                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
-                    validator=validate_input_file_dummy,
-                    splitter=split_data_by_size,
-                ),
-            ],
-            output_slots=[
-                OutputSlot(
-                    name="step_3_main_output",
-                    aggregator=concatenate_datasets,
-                ),
-            ],
+            step=Step(
+                step_name="step_3",
+                input_slots=[
+                    InputSlot(
+                        name="step_3_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                        splitter=split_data_by_size,
+                    ),
+                ],
+                output_slots=[
+                    OutputSlot(
+                        name="step_3_main_output",
+                        aggregator=concatenate_datasets,
+                    ),
+                ],
+            ),
         ),
         self_edges=[
             EdgeParams(

{easylink-0.1.8 → easylink-0.1.10}/src/easylink/step.py

@@ -1138,19 +1138,25 @@ class EmbarrassinglyParallelStep(Step):
 
     An ``EmbarrassinglyParallelStep`` is different than a :class:`ParallelStep`
     in that it is not configured by the user to be run in parallel - it completely
-    happens on the back end for performance reasons. As such, note that it inherits
-    from :class:`Step` instead of :class:`TemplatedStep`.
+    happens on the back end for performance reasons.
 
     See :class:`Step` for inherited attributes.
+
+    Parameters
+    ----------
+    step
+        The ``Step`` to be run in an embarrassingly parallel manner. To run multiple
+        steps in parallel, use a :class:`HierarchicalStep`.
+
     """
 
     def __init__(
         self,
-        step_name: str,
-        input_slots: Iterable[InputSlot],
-        output_slots: Iterable[OutputSlot],
+        step: Step,
     ) -> None:
-        super().__init__(step_name, input_slots=input_slots, output_slots=output_slots)
+        super().__init__(
+            step.step_name, step.name, step.input_slots.values(), step.output_slots.values()
+        )
         self._validate()
 
     def _validate(self) -> None:

{easylink-0.1.8 → easylink-0.1.10/src/easylink.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.8
+Version: 0.1.10
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.8 → easylink-0.1.10}/src/easylink.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ docs/source/api_reference/utilities/paths.rst
 docs/source/api_reference/utilities/splitter_utils.rst
 docs/source/api_reference/utilities/validation_utils.rst
 docs/source/concepts/index.rst
+docs/source/concepts/pipeline_schema/index.rst
 docs/source/user_guide/cli.rst
 docs/source/user_guide/index.rst
 docs/source/user_guide/tutorials/getting_started.rst

{easylink-0.1.8 → easylink-0.1.10}/tests/unit/test_graph_components.py

@@ -7,7 +7,6 @@ from easylink.graph_components import (
     InputSlotMapping,
     OutputSlot,
     OutputSlotMapping,
-    SlotMapping,
     StepGraph,
 )
 from easylink.implementation import Implementation
@@ -29,6 +28,93 @@ def test_output_slot() -> None:
     assert output_slot.name == "file1"
 
 
+def test_input_slot_hashing() -> None:
+    slot = InputSlot("slot", "foo", validate_input_file_dummy)
+    slot_dupe = InputSlot("slot", "foo", validate_input_file_dummy)
+    assert slot == slot_dupe
+    assert {slot, slot_dupe} == {slot}
+
+    slot_with_splitter = InputSlot("slot", "foo", validate_input_file_dummy, dummy_splitter)
+
+    assert slot != slot_with_splitter
+    assert {slot, slot_with_splitter, slot, slot_with_splitter} == {
+        slot,
+        slot_with_splitter,
+    }
+
+    slot_with_different_splitter = InputSlot(
+        "slot", "foo", validate_input_file_dummy, dummy_splitter_2
+    )
+    assert slot_with_splitter != slot_with_different_splitter
+    assert {
+        slot,
+        slot_dupe,
+        slot_with_splitter,
+        slot_with_different_splitter,
+        slot_with_splitter,
+        slot_with_different_splitter,
+    } == {
+        slot,
+        slot_with_splitter,
+        slot_with_different_splitter,
+    }
+
+    slot_with_different_validator = InputSlot("slot", "foo", dummy_validator)
+    assert slot != slot_with_different_validator
+    assert {slot, slot_dupe, slot_with_different_validator} == {
+        slot,
+        slot_with_different_validator,
+    }
+
+
+def test_output_slot_hashing() -> None:
+    slot = OutputSlot("slot")
+    slot_dupe = OutputSlot("slot")
+    assert slot == slot_dupe
+    assert {slot, slot_dupe} == {slot}
+
+    slot_with_aggregator = OutputSlot("slot", dummy_aggregator)
+
+    assert slot != slot_with_aggregator
+    assert {slot, slot_with_aggregator, slot, slot_with_aggregator} == {
+        slot,
+        slot_with_aggregator,
+    }
+
+    slot_with_different_aggregator = OutputSlot("slot", dummy_aggregator_2)
+    assert slot_with_aggregator != slot_with_different_aggregator
+    assert {
+        slot,
+        slot_dupe,
+        slot_with_aggregator,
+        slot_with_different_aggregator,
+        slot_with_aggregator,
+        slot_with_different_aggregator,
+    } == {
+        slot,
+        slot_with_aggregator,
+        slot_with_different_aggregator,
+    }
+
+    input_slot = InputSlot("slot", "foo", validate_input_file_dummy)
+
+
+def test_slot_mutability() -> None:
+    slot = InputSlot("slot", "foo", validate_input_file_dummy)
+    assert slot.splitter is None
+    slot.splitter = dummy_splitter
+    assert slot.splitter == dummy_splitter
+    slot.splitter = dummy_splitter_2
+    assert slot.splitter == dummy_splitter_2
+
+    slot = OutputSlot("slot")
+    assert slot.aggregator is None
+    slot.aggregator = dummy_aggregator
+    assert slot.aggregator == dummy_aggregator
+    slot.aggregator = dummy_aggregator_2
+    assert slot.aggregator == dummy_aggregator_2
+
+
 def test_edge() -> None:
     edge = EdgeParams(
         source_node="input_data",
@@ -136,3 +222,28 @@ def test_output_slot_mapping() -> None:
     assert new_edge.target_node == "output_data"
     assert new_edge.output_slot == "step_1a_main_input"
     assert new_edge.input_slot == "file1"
+
+
+####################
+# Helper functions #
+####################
+
+
+def dummy_splitter():
+    pass
+
+
+def dummy_splitter_2():
+    pass
+
+
+def dummy_aggregator():
+    pass
+
+
+def dummy_aggregator_2():
+    pass
+
+
+def dummy_validator():
+    pass

{easylink-0.1.8 → easylink-0.1.10}/tests/unit/test_pipeline_graph.py

@@ -99,24 +99,42 @@ def test_implementations(default_config: Config) -> None:
             "input_slot",
             False,
             {
-                ("step_a", InputSlot("foo", env_var="bar", validator=None)),
-                ("step_b", InputSlot("baz", env_var="spam", validator=None)),
+                (
+                    "step_a",
+                    InputSlot("foo", env_var="bar", validator=validate_input_file_dummy),
+                ),
+                (
+                    "step_b",
+                    InputSlot("baz", env_var="spam", validator=validate_input_file_dummy),
+                ),
             },
         ),
         (
             "input_slot",
             True,
             {
-                ("step_a", InputSlot("foo", env_var="bar", validator=None)),
-                ("step_b", InputSlot("foo", env_var="spam", validator=None)),
+                (
+                    "step_a",
+                    InputSlot("foo", env_var="bar", validator=validate_input_file_dummy),
+                ),
+                (
+                    "step_b",
+                    InputSlot("foo", env_var="spam", validator=validate_input_file_dummy),
+                ),
             },
         ),
         (
             "input_slot",
             True,
             {
-                ("step_a", InputSlot("foo", env_var="bar", validator=None)),
-                ("step_b", InputSlot("baz", env_var="bar", validator=None)),
+                (
+                    "step_a",
+                    InputSlot("foo", env_var="bar", validator=validate_input_file_dummy),
+                ),
+                (
+                    "step_b",
+                    InputSlot("baz", env_var="bar", validator=validate_input_file_dummy),
+                ),
             },
         ),
         (

{easylink-0.1.8 → easylink-0.1.10}/tests/unit/test_step.py

@@ -1114,16 +1114,18 @@ def test_complex_choice_step_get_implementation_graph(
 @pytest.fixture
 def embarrassingly_parallel_step_params() -> dict[str, Any]:
     return {
-        "step_name": "step_3",
-        "input_slots": [
-            InputSlot(
-                "step_3_main_input",
-                "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
-                validate_input_file_dummy,
-                split_data_by_size,
-            )
-        ],
-        "output_slots": [OutputSlot("step_3_main_output", concatenate_datasets)],
+        "step": Step(
+            step_name="step_3",
+            input_slots=[
+                InputSlot(
+                    "step_3_main_input",
+                    "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validate_input_file_dummy,
+                    split_data_by_size,
+                )
+            ],
+            output_slots=[OutputSlot("step_3_main_output", concatenate_datasets)],
+        ),
     }
 
 
@@ -1261,9 +1263,11 @@ def test_embarrassingly_parallel_step__validation(
     expected_error_msg: str | list[str],
 ):
     step_params = {
-        "step_name": "step",
-        "input_slots": input_slots,
-        "output_slots": output_slots,
+        "step": Step(
+            step_name="step",
+            input_slots=input_slots,
+            output_slots=output_slots,
+        ),
     }
     with pytest.raises(ValueError) as error:
         EmbarrassinglyParallelStep(**step_params)

easylink-0.1.8/src/easylink/_version.py

@@ -1 +0,0 @@
-__version__ = "0.1.8"