easylink 0.1.21__py3-none-any.whl → 0.1.22__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.21"
+__version__ = "0.1.22"

easylink/configuration.py

@@ -184,7 +184,9 @@ class Config(LayeredConfigTree):
     #################
 
     def _get_schema(self, schema_name: str = "main") -> PipelineSchema:
-        """Returns the first :class:`~easylink.pipeline_schema.PipelineSchema` that validates the requested pipeline.
+        """Gets the requested :class:`~easylink.pipeline_schema.PipelineSchema`.
+
+        The schema is only returned if it validates the pipeline configuration.
 
         Parameters
         ----------
@@ -205,11 +207,10 @@ class Config(LayeredConfigTree):
         Notes
         -----
         This acts as the pipeline configuration file's validation method since
-        we can only find a matching ``PipelineSchema`` if that file is valid.
+        we can only validate the ``PipelineSchema`` if that file is valid.
 
         """
         errors = defaultdict(dict)
-        # Try each schema until one is validated
         schema = PipelineSchema.get_schema(schema_name)
         logs = schema.validate_step(self.pipeline, self.input_data)
         if logs:

easylink/pipeline_schema.py

@@ -159,10 +159,10 @@ class PipelineSchema(HierarchicalStep):
         )
 
     @classmethod
-    def get_schema(cls, name: str = "main") -> list["PipelineSchema"]:
-        """Gets all allowable ``PipelineSchemas``.
+    def get_schema(cls, name: str = "main") -> "PipelineSchema":
+        """Gets the requested ``PipelineSchema``.
 
-        These ``PipelineSchemas`` represent the fully supported pipelines and are
+        This ``PipelineSchema`` represents the fully supported pipelines and is
         used to validate the user-requested pipeline.
 
         Parameters

easylink/pipeline_schema_constants/__init__.py

@@ -27,4 +27,5 @@ SCHEMA_PARAMS = {
     "auto_parallel_cloneable_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_CLONEABLE_STEP,
     "auto_parallel_loop_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_LOOP_STEP,
     "auto_parallel_hierarchical_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_HIERARCHICAL_STEP,
+    "default_implementations": testing.SCHEMA_PARAMS_DEFAULT_IMPLEMENTATIONS,
 }

easylink/pipeline_schema_constants/testing.py

@@ -640,3 +640,126 @@ EDGES_OUTPUT_DIR = [
     ),
 ]
 SCHEMA_PARAMS_OUTPUT_DIR = (NODES_OUTPUT_DIR, EDGES_OUTPUT_DIR)
+
+
+NODES_DEFAULT_IMPLEMENTATIONS = [
+    InputStep(),
+    HierarchicalStep(
+        step_name="step_1",
+        input_slots=[
+            InputSlot(
+                name="step_1_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            ),
+        ],
+        output_slots=[OutputSlot("step_1_main_output")],
+        nodes=[
+            Step(
+                step_name="step_1a",
+                input_slots=[
+                    InputSlot(
+                        name="step_1a_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                    ),
+                ],
+                output_slots=[OutputSlot("step_1a_main_output")],
+                default_implementation="step_1a_python_pandas",
+            ),
+            Step(
+                step_name="step_1b",
+                input_slots=[
+                    InputSlot(
+                        name="step_1b_main_input",
+                        env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                        validator=validate_input_file_dummy,
+                    ),
+                ],
+                output_slots=[OutputSlot("step_1b_main_output")],
+                default_implementation="step_1b_python_pandas",
+            ),
+        ],
+        edges=[
+            EdgeParams(
+                source_node="step_1a",
+                target_node="step_1b",
+                output_slot="step_1a_main_output",
+                input_slot="step_1b_main_input",
+            ),
+        ],
+        input_slot_mappings=[
+            InputSlotMapping(
+                parent_slot="step_1_main_input",
+                child_node="step_1a",
+                child_slot="step_1a_main_input",
+            ),
+        ],
+        output_slot_mappings=[
+            OutputSlotMapping(
+                parent_slot="step_1_main_output",
+                child_node="step_1b",
+                child_slot="step_1b_main_output",
+            ),
+        ],
+        default_implementation="step_1_python_pandas",
+    ),
+    Step(
+        step_name="step_2",
+        input_slots=[
+            InputSlot(
+                name="step_2_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            )
+        ],
+        output_slots=[OutputSlot("step_2_main_output")],
+        default_implementation="step_2_python_pandas",
+    ),
+    LoopStep(
+        template_step=Step(
+            step_name="step_3",
+            input_slots=[
+                InputSlot(
+                    name="step_3_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                )
+            ],
+            output_slots=[OutputSlot("step_3_main_output")],
+        ),
+        self_edges=[
+            EdgeParams(
+                source_node="step_3",
+                target_node="step_3",
+                output_slot="step_3_main_output",
+                input_slot="step_3_main_input",
+            ),
+        ],
+        default_implementation="step_3_python_pandas",
+    ),
+    CloneableStep(
+        template_step=Step(
+            step_name="step_4",
+            input_slots=[
+                InputSlot(
+                    name="step_4_main_input",
+                    env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                    validator=validate_input_file_dummy,
+                ),
+            ],
+            output_slots=[
+                OutputSlot(
+                    name="step_4_main_output",
+                ),
+            ],
+        ),
+        default_implementation="step_4_python_pandas",
+    ),
+    OutputStep(
+        input_slots=[
+            InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
+        ],
+    ),
+]
+SCHEMA_PARAMS_DEFAULT_IMPLEMENTATIONS = (NODES_DEFAULT_IMPLEMENTATIONS, EDGES_TWO_STEPS)

easylink/step.py

@@ -92,6 +92,7 @@ class Step:
         input_slot_mappings: Iterable[InputSlotMapping] = (),
         output_slot_mappings: Iterable[OutputSlotMapping] = (),
         is_auto_parallel: bool = False,
+        default_implementation: str | None = None,
     ) -> None:
         if not step_name and not name:
             raise ValueError("All Steps must contain a step_name, name, or both.")
@@ -127,6 +128,9 @@ class Step:
         ``OutputSlotMappings`` of this ``Step``."""
         self.is_auto_parallel = is_auto_parallel
         """Whether or not this ``Step`` is to be automatically run in parallel."""
+        self.default_implementation = default_implementation
+        """The default implementation to use for this ``Step`` if the ``Step`` is
+        not explicitly configured in the pipeline specification."""
         self.parent_step = None
         """This ``Step's`` parent ``Step``, if applicable."""
         self._configuration_state = None
@@ -580,6 +584,7 @@ class HierarchicalStep(Step):
         input_slot_mappings=(),
         output_slot_mappings=(),
         directly_implemented=True,
+        default_implementation: str | None = None,
     ):
         super().__init__(
             step_name,
@@ -588,6 +593,7 @@ class HierarchicalStep(Step):
             output_slots,
             input_slot_mappings,
             output_slot_mappings,
+            default_implementation=default_implementation,
         )
         self.nodes = nodes
         """All sub-nodes (i.e. sub-``Steps``) that make up this ``HierarchicalStep``."""
@@ -722,13 +728,19 @@ class HierarchicalStep(Step):
             step = self.step_graph.nodes[node]["step"]
             if isinstance(step, IOStep):
                 continue
+            if step.name not in step_config:
+                default_implementation = self.step_graph.nodes[step.name][
+                    "step"
+                ].default_implementation
+                step_errors = (
+                    {f"step {step.name}": ["The step is not configured."]}
+                    if not default_implementation
+                    else {}
+                )
             else:
-                if step.name not in step_config:
-                    step_errors = {f"step {step.name}": ["The step is not configured."]}
-                else:
-                    step_errors = step.validate_step(
-                        step_config[step.name], combined_implementations, input_data_config
-                    )
+                step_errors = step.validate_step(
+                    step_config[step.name], combined_implementations, input_data_config
+                )
             if step_errors:
                 errors.update(step_errors)
         extra_steps = set(step_config.keys()) - set(self.step_graph.nodes)
@@ -830,12 +842,14 @@ class TemplatedStep(Step, ABC):
     def __init__(
         self,
         template_step: Step,
+        default_implementation: str | None = None,
     ) -> None:
         super().__init__(
             template_step.step_name,
             template_step.name,
             template_step.input_slots.values(),
             template_step.output_slots.values(),
+            default_implementation=default_implementation,
         )
         self.step_graph = None
         """The :class:`~easylink.graph_components.StepGraph` i.e. the directed acyclic 
@@ -1110,8 +1124,9 @@ class LoopStep(TemplatedStep):
         self,
         template_step: Step | None = None,
         self_edges: Iterable[EdgeParams] = (),
+        default_implementation: str | None = None,
     ) -> None:
-        super().__init__(template_step)
+        super().__init__(template_step, default_implementation)
         self.self_edges = self_edges
         """:class:`~easylink.graph_components.EdgeParams` that represent self-edges,
         i.e. edges that connect the output of one loop to the input of the next."""
@@ -2181,15 +2196,32 @@ class NonLeafConfigurationState(ConfigurationState):
 
         This method recursively traverses the ``StepGraph`` and sets the configuration
         state for each ``Step`` until reaching all leaf nodes.
+
+        Notes
+        -----
+        If a ``Step`` name is missing from the ``step_config``, we know that it
+        must have a default implementation because we already validated that one
+        exists during :meth:`HierarchicalStep._validate_step_graph`. In that case,
+        we manually instantiate and use a ``step_config`` with the default implementation.
         """
         for sub_node in self._step.step_graph.nodes:
             sub_step = self._step.step_graph.nodes[sub_node]["step"]
-            # IOSteps, SplitterSteps, and AggregatorSteps never appear explicitly in the configuration
-            step_config = (
-                self.step_config
-                if isinstance(sub_step, (IOStep, SplitterStep, AggregatorStep))
-                else self.step_config[sub_step.name]
-            )
+            try:
+                step_config = (
+                    self.step_config
+                    if isinstance(sub_step, StandaloneStep)
+                    else self.step_config[sub_step.name]
+                )
+            except KeyError:
+                # We know that any missing keys must have a default implementation
+                # (because we have already checked that it exists during validation)
+                step_config = LayeredConfigTree(
+                    {
+                        "implementation": {
+                            "name": sub_step.default_implementation,
+                        }
+                    }
+                )
             sub_step.set_configuration_state(
                 step_config, self.combined_implementations, self.input_data_config
             )

{easylink-0.1.21.dist-info → easylink-0.1.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.21
+Version: 0.1.22
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.21.dist-info → easylink-0.1.22.dist-info}/RECORD

@@ -1,24 +1,22 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=qEmNtjnOwhDYQ0cHPPtUkUaghzD2xl0thJEznl4giYw,23
+easylink/_version.py,sha256=zmP2TRnzKPjZJ1eiBcT-cRInsji6FW-OVD3FafQFCc4,23
 easylink/cli.py,sha256=zQO4lOVoZ3eVgPVWT2sCF4zNoKgiDJP9ReTh2Myr9jc,10307
-easylink/configuration.py,sha256=hgmG5SIbYqnHDHfk44Gr3QX7C3yTaEVW6GuKeMqvu6c,12689
+easylink/configuration.py,sha256=TPd3WbqUcJMJDPJuHeo5ZebvZPQrRyfm8-laad2sOFk,12681
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
 easylink/implementation.py,sha256=lSF37g-aQYgdLI0lgFaDrBkp23te9O9B1V-CmmRtB-0,14514
 easylink/implementation_metadata.yaml,sha256=GoU_aWjVryG8-xjUHkC2nCUeznmYD0BwfJYnNrpZ8P4,10670
 easylink/pipeline.py,sha256=NJyMInbOCjJ_5kRzzuK5AcupvC7ecd5qLOC-f1Gy3Ss,17701
 easylink/pipeline_graph.py,sha256=jtjS7_2IVa189U8cL621600erC_0pa6BKPRRO8VqwiU,23760
-easylink/pipeline_schema.py,sha256=FieJBa3rKgaCIB9QDuQEfWJ9joNBUUp6iHT6xmns-Vk,6886
+easylink/pipeline_schema.py,sha256=sj6YQqMFUS35RRFWdlrhPlud68d3Rb5y7iKfAEZy1qs,6877
 easylink/rule.py,sha256=QJPmrvQUZPnqGFD9UmMK8imdJ7VODzGlUOSnpJhb9AU,16677
 easylink/runner.py,sha256=Z9GKgiUAWtp0dW1cyAp86MGthIDeABJtHDXUtzv0-kE,6714
-easylink/step.py,sha256=SqOxinHyRaLCEnB_y5dvhGMaRLyphQDCpVsQ3160c9U,89588
+easylink/step.py,sha256=zQAoz4HlSVvgS7iMlfmCrXluOtPQxbSgPZOeyZwjdpo,91085
 easylink/devtools/implementation_creator.py,sha256=gZZpfpiOOh912nNjR_d5wR0mBO5c09jWMS0FSYRcE1o,19120
-easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
-easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=SMNXz49DSwx05PHMKUsunJsgMOqsBJaAHA1fmIOJsUU,1445
+easylink/pipeline_schema_constants/__init__.py,sha256=njUL2v67WFC3dW_FjOXWRLSeOhInox_ZINLEji0_7V8,1523
 easylink/pipeline_schema_constants/development.py,sha256=0x6lWqBmP1K9AebEmeZ4veSnLBcdQcZXzbV6lCU11bc,12622
 easylink/pipeline_schema_constants/main.py,sha256=kcAhdbK_BhS79LUMhKkn-Uttl2vA-MHVX4M1StTloJQ,22934
-easylink/pipeline_schema_constants/testing.py,sha256=G7szRMyY48dL8kUHWq2MeMaV2G0F-AdAPsQxFzdUnFI,20567
+easylink/pipeline_schema_constants/testing.py,sha256=ZFD19CpcidZPVUYBvh8LAa5sZEERT2yfoFa-3xmskFs,24595
 easylink/steps/cascading/exclude_clustered.def,sha256=GfoDqO2Vtsh7VI8SwGaJtv_KtKjs-UmBcivqQ7OPkjk,503
 easylink/steps/cascading/exclude_clustered.py,sha256=NSA6GZBzGa7e6CH4tacCGfr0Y9sUM29g9Nf8NquHB44,2612
 easylink/steps/cascading/exclude_none.def,sha256=iFUhUMocxtkA0NErkjVrBxY0MUdS3DIPNsbCpTJRP0k,488
@@ -83,9 +81,9 @@ easylink/utilities/paths.py,sha256=9inDKMPorAaWh5vhjc3j1Tj_aXVKhLdodiJO9H1nNes,9
 easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=KXiVUYJ9TGxCQmrwos18pB1sxG_0Ay67qoDJT6vem2o,3828
 easylink/utilities/validation_utils.py,sha256=DBJB2TLVklgYw1WaaPse9vqtwPLMGmZNYM2cbCZsoHI,18417
-easylink-0.1.21.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
-easylink-0.1.21.dist-info/METADATA,sha256=wdHGbqg2d4yte9ep9mO_GAr2EbUmEAVHHjPg6LsvMLE,3565
-easylink-0.1.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-easylink-0.1.21.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.21.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.21.dist-info/RECORD,,
+easylink-0.1.22.dist-info/licenses/LICENSE,sha256=z6NBo3hY2Pbh2Wjm9IcHLyQoG-kd816uASh8PbwhTcQ,1530
+easylink-0.1.22.dist-info/METADATA,sha256=hei9KKa0HUgy1Z4aU-nPEAs8KF2_TEe7J0-_esdCG40,3565
+easylink-0.1.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+easylink-0.1.22.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.22.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.22.dist-info/RECORD,,

easylink/images/spark_cluster/Dockerfile

@@ -1,16 +0,0 @@
-# Stage 1: Start with the miniconda3 base image
-FROM continuumio/miniconda3 as conda-base
-
-# Create a new conda environment
-SHELL ["/bin/bash", "--login", "-c"]
-RUN conda init bash \
-    && . ~/.bashrc \
-    && conda create -n spark_cluster python=3.10
-
-# Stage 2: Start with the Apache Spark base image
-FROM apache/spark@sha256:a1dd2487a97fb5e35c5a5b409e830b501a92919029c62f9a559b13c4f5c50f63 as spark-base
-
-COPY --from=conda-base /opt/conda /opt/conda
-
-# Set PATH for conda environment and conda itself
-ENV PATH=/opt/conda/envs/spark_cluster/bin:/opt/conda/condabin:${PATH}

easylink/images/spark_cluster/README.md

@@ -1,15 +0,0 @@
-# spark_cluster container
-NOTE: Spinning up a spark cluster using `easylink` currently requires building an image from this directory.
-
-This is done by running the following commands from this directory:
-
-```
-# build the image
-$ sudo docker build -t easylink:sparkbuilder .
-# save as compressed tarball
-$ sudo docker save easylink:sparkbuilder | gzip > spark_cluster.tar.gz
-# remove the image
-$ sudo docker rmi easylink:sparkbuilder
-# convert the image from the docker image
-$ singularity build --force spark_cluster.sif docker-archive://$(pwd)/spark_cluster.tar.gz
-```