easylink 0.1.14__py3-none-any.whl → 0.1.16__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.14"
+__version__ = "0.1.16"

easylink/cli.py

@@ -41,17 +41,21 @@ As before, refer to ``easylink generate-dag --help`` for information on other op
 For usage documentation, see :ref:`cli`.
 """
 
+import os
 from collections.abc import Callable
+from pathlib import Path
 
 import click
 from loguru import logger
 
 from easylink import runner
+from easylink.devtools import implementation_creator
 from easylink.utilities.data_utils import get_results_directory
 from easylink.utilities.general_utils import (
     configure_logging_to_terminal,
     handle_exceptions,
 )
+from easylink.utilities.paths import CONTAINER_DIR
 
 SHARED_OPTIONS = [
     click.option(
@@ -87,6 +91,9 @@ SHARED_OPTIONS = [
         default=False,
         help="Do not save the results in a timestamped sub-directory of ``--output-dir``.",
     ),
+]
+
+VERBOSE_WITH_DEBUGGER_OPTIONS = [
     click.option(
         "-v", "--verbose", count=True, help="Increase logging verbosity.", hidden=True
     ),
@@ -100,12 +107,26 @@ SHARED_OPTIONS = [
 ]
 
 
+def _pass_verbose_with_debugger_options(func: Callable) -> Callable:
+    """Passes verbosity and debugger options to a click command.
+
+    Parameters
+    ----------
+    func
+        The click command function to add shared options to.
+
+    Returns
+    -------
+        The click command function with the shared options added.
+    """
+    for option in VERBOSE_WITH_DEBUGGER_OPTIONS:
+        func = option(func)
+    return func
+
+
 def _pass_shared_options(func: Callable) -> Callable:
     """Passes shared options to a click command.
 
-    This function is a decorator that takes a click command callable and adds the
-    shared options defined in ``SHARED_OPTIONS`` to it.
-
     Parameters
     ----------
     func
@@ -115,7 +136,7 @@ def _pass_shared_options(func: Callable) -> Callable:
     -------
         The click command function with the shared options added.
     """
-    for option in SHARED_OPTIONS:
+    for option in SHARED_OPTIONS + VERBOSE_WITH_DEBUGGER_OPTIONS:
         func = option(func)
     return func
 
@@ -204,3 +225,90 @@ def generate_dag(
         results_dir=results_dir,
     )
     logger.info("*** DAG saved to result directory ***")
+
+
+#####################
+# Development tools #
+#####################
+
+
+@click.group(hidden=True)
+def devtools():
+    """Development tools for EasyLink."""
+    pass
+
+
+easylink.add_command(devtools)
+
+
+@devtools.command()
+@_pass_verbose_with_debugger_options
+@click.argument(
+    "scripts",
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True),
+    nargs=-1,
+)
+@click.option(
+    "-o",
+    "--output-dir",
+    type=click.Path(exists=False, dir_okay=True, file_okay=False, resolve_path=True),
+    help=(
+        "The directory to move the container to. If no value is passed, it will "
+        f"be moved to {CONTAINER_DIR} in a sub-directory named with the username."
+    ),
+)
+def create_implementation(
+    scripts: tuple[str, ...],
+    output_dir: str | None,
+    verbose: int,
+    with_debugger: bool,
+):
+    """Creates EasyLink implementations from implementation details.
+
+    This is a helper tool for developers to more easily create implementations
+    and register them with the EasyLink framework.
+
+    SCRIPTS are the filepaths to the implementation Python scripts to be run from within
+    a newly created container. Each script must specify (1) the name of the pipeline
+    step that it is implementing as well as, optionally, (2) any required pypi dependencies,
+    and (3) the pipeline schema that that the step the script implements is part of
+    (will default to "main" if not specified).
+
+    These values are to be specified in the script using comments with the exact
+    format shown in the example below.
+
+        # STEP_NAME: blocking
+
+        # REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+        # PIPELINE_SCHEMA: development
+
+    Note that the requirements should be formatted as a single line.
+
+    If an implementation of the same name already exists, it will be overwritten
+    automatically and the new one registered with EasyLink.
+    """
+    if not scripts:
+        logger.error("No scripts provided.")
+        return
+    output_dir = Path(output_dir) if output_dir else Path(f"{CONTAINER_DIR}/{os.getlogin()}")
+    if not output_dir.exists():
+        # make the directory with rwxrwxr-x permissions
+        output_dir.mkdir(parents=True, mode=0o775)
+    if not output_dir.exists():
+        raise FileNotFoundError(
+            f"Output directory {output_dir} does not exist and could not be created."
+        )
+    configure_logging_to_terminal(verbose)
+    main = handle_exceptions(
+        func=implementation_creator.main,
+        exceptions_logger=logger,
+        with_debugger=with_debugger,
+    )
+    list_str = ""
+    for script in scripts:
+        script = Path(script)
+        logger.info(f"Creating implementation for {script.name}")
+        main(script_path=script, host=output_dir)
+        list_str += f"  - {script.stem}\n"
+    logger.info("*** Implementations created ***\n" f"{list_str}")

easylink/devtools/implementation_creator.py

@@ -0,0 +1,435 @@
+"""
+An EasyLink "implementation" (related to, but not to be confused with the
+:class:`~easylink.implementation.Implementation` class object) is, at the most
+basic level, a container that implements some step of the pipeline as well as
+other supporting information to connect said container to the EasyLink framework.
+
+In order to create an implementation, three things are needed:
+1. The container that runs the script that implements a step of the pipeline must be created.
+2. The container must be moved to the proper hosting location so EasyLink can find it.
+3. The container must be registered with EasyLink so it can be used.
+
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+from typing import cast
+
+import yaml
+from loguru import logger
+
+from easylink.pipeline_schema_constants import ALLOWED_SCHEMA_PARAMS
+from easylink.step import (
+    ChoiceStep,
+    EmbarrassinglyParallelStep,
+    HierarchicalStep,
+    IOStep,
+    Step,
+    TemplatedStep,
+)
+from easylink.utilities.data_utils import load_yaml
+from easylink.utilities.paths import IMPLEMENTATION_METADATA
+
+
+def main(script_path: Path, host: Path) -> None:
+    """Creates a container to run a specific script and registers it with EasyLink.
+
+    Parameters
+    ----------
+    script_path
+        The filepath to a single script that implements a step of the pipeline.
+    host
+        The host directory to move the container to.
+    """
+    creator = ImplementationCreator(script_path, host)
+    creator.create_recipe()
+    creator.build_container()
+    creator.move_container()
+    creator.register()
+
+
+class ImplementationCreator:
+    """A class used to create a container for a specific implementation.
+
+    Parameters
+    ----------
+    script_path
+        The filepath to a single script that implements a step of the pipeline.
+    host
+        The host directory to move the container to.
+    recipe_path
+        The filepath to the recipe file that will be created. It will be created
+        in the same directory as the script.
+    local_container_path
+        The filepath to the local container that will be created. It will be created
+        in the same directory as the script.
+    hosted_container_path
+        The filepath to to move the container to. This is where EasyLink will look
+        for the container.
+    implementation_name
+        The name of the implementation. It is by definition the name of the script.
+    requirements
+        The install requirements for the implementation (if any).
+    step
+        The name of the step that this implementation implements.
+    output_slot
+        The name of the output slot that this implementation sends results to.
+    """
+
+    def __init__(self, script_path: Path, host: Path) -> None:
+        self.script_path = script_path
+        """The filepath to a single script that implements a step of the pipeline."""
+        self.host = host
+        """The host directory to move the container to."""
+        self.recipe_path = script_path.with_suffix(".def")
+        """The filepath to the recipe file that will be created. It will be created
+        in the same directory as the script."""
+        self.local_container_path = script_path.with_suffix(".sif")
+        """The filepath to the local container that will be created. It will be created
+        in the same directory as the script."""
+        self.hosted_container_path = self.host / self.local_container_path.name
+        """The filepath to to move the container to. This is where EasyLink will look
+        for the container."""
+        self.implementation_name = script_path.stem
+        """The name of the implementation. It is by definition the name of the script."""
+        self.requirements = self._extract_requirements(script_path)
+        """The install requirements for the implementation (if any)."""
+        self.step = self._extract_implemented_step(script_path)
+        """The name of the step that this implementation implements."""
+        self.output_slot = self._extract_output_slot(script_path, self.step)
+        """The name of the output slot that this implementation sends results to."""
+
+    def create_recipe(self) -> None:
+        """Builds the singularity recipe and writes it to disk."""
+
+        recipe = PythonRecipe(self.script_path, self.recipe_path, self.requirements)
+        recipe.build()
+        recipe.write()
+        pass
+
+    def build_container(self) -> None:
+        """Builds the container from the recipe.
+
+        Raises
+        ------
+        subprocess.CalledProcessError
+            If the subprocess fails.
+        Exception
+            If the container fails to build for any reason.
+        """
+        logger.info(f"Building container for '{self.implementation_name}'")
+        if self.local_container_path.exists():
+            logger.warning(
+                f"Container {self.local_container_path} already exists. Overwriting it."
+            )
+
+        try:
+            cmd = [
+                "singularity",
+                "build",
+                "--remote",
+                "--force",
+                str(self.local_container_path),
+                str(self.recipe_path),
+            ]
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                cwd=self.local_container_path.parent,
+            )
+
+            # stream output to console
+            for line in process.stdout:  # type: ignore[union-attr]
+                print(line, end="")
+            process.wait()
+
+            if process.returncode == 0:
+                logger.info(
+                    f"Successfully built container '{self.local_container_path.name}'"
+                )
+            else:
+                logger.error(
+                    f"Failed to build container '{self.local_container_path.name}'. "
+                    f"Error: {process.returncode}"
+                )
+                raise subprocess.CalledProcessError(
+                    process.returncode, cmd, output=process.stderr.read()  # type: ignore[union-attr]
+                )
+        except Exception as e:
+            logger.error(
+                f"Failed to build container '{self.local_container_path.name}'. "
+                f"Error: {e}"
+            )
+            raise
+
+    def move_container(self) -> None:
+        """Moves the container to the proper location for EasyLink to find it."""
+        logger.info(f"Moving container '{self.implementation_name}' to {self.host}")
+        if self.hosted_container_path.exists():
+            logger.warning(
+                f"Container {self.hosted_container_path} already exists. Overwriting it."
+            )
+        shutil.move(str(self.local_container_path), str(self.hosted_container_path))
+
+    def register(self) -> None:
+        """Registers the container with EasyLink.
+
+        Specifically, this function adds the implementation details to the
+        implementation_metadata.yaml registry file.
+        """
+        logger.info(f"Registering container '{self.implementation_name}'")
+        info = load_yaml(IMPLEMENTATION_METADATA)
+        if self.implementation_name in info:
+            logger.warning(
+                f"Implementation '{self.implementation_name}' already exists in the registry. "
+                "Overwriting it with the latest data."
+            )
+        info[self.implementation_name] = {
+            "steps": [self.step],
+            "image_path": str(self.hosted_container_path),
+            "script_cmd": f"python /{self.script_path.name}",
+            "outputs": {
+                self.output_slot: "result.parquet",
+            },
+        }
+        self._write_metadata(info)
+
+    @staticmethod
+    def _extract_requirements(script_path: Path) -> str:
+        """Extracts the script's dependency requirements (if any).
+
+        The expectation is that any requirements are specified within the script
+        as a comment of the format:
+
+        .. code-block:: python
+            # REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+        This is an optional field and only required if the script actually has dependencies.
+
+        The requirements must be specified as a single space-separated line.
+        """
+        requirements = _extract_metadata("REQUIREMENTS", script_path)
+        if len(requirements) == 0:
+            logger.info(f"No requirements found in {script_path}.")
+            requirements.append("")
+        return requirements[0]
+
+    @staticmethod
+    def _extract_implemented_step(script_path: Path) -> str:
+        """Extracts the name of the step that this script is implementing.
+
+        The expectation is that the step's name is specified within the script
+        as a comment of the format:
+
+        .. code-block:: python
+            # STEP_NAME: blocking
+        """
+        step_info = _extract_metadata("STEP_NAME", script_path)
+        if len(step_info) == 0:
+            raise ValueError(
+                f"Could not find a step name in {script_path}. "
+                "Please ensure the script contains a comment of the form '# STEP_NAME: <name>'"
+            )
+        steps = [step.strip() for step in step_info[0].split(",")]
+        if len(steps) > 1:
+            raise NotImplementedError(
+                f"Multiple steps are not yet supported. {script_path} is requesting "
+                f"to implement {steps}."
+            )
+        return steps[0]
+
+    @staticmethod
+    def _extract_output_slot(script_path: Path, step_name: str) -> str:
+        """Extracts the name of the output slot that this script is implementing."""
+        schema = ImplementationCreator._extract_pipeline_schema(script_path)
+        implementable_steps = ImplementationCreator._extract_implementable_steps(schema)
+        step_names = [step.name for step in implementable_steps]
+        if step_name not in step_names:
+            raise ValueError(
+                f"'{step_name}' does not exist as an implementable step in the '{schema}' pipeline schema. "
+            )
+        duplicates = list(set([step for step in step_names if step_names.count(step) > 1]))
+        if duplicates:
+            raise ValueError(
+                f"Multiple implementable steps with the same name found in the '{schema}' "
+                f"pipeline schema: {duplicates}."
+            )
+        implemented_step = [step for step in implementable_steps if step.name == step_name][0]
+        if len(implemented_step.output_slots) != 1:
+            raise NotImplementedError(
+                f"Multiple output slots are not yet supported. {script_path} is requesting "
+                f"to implement {step_name} with {len(implemented_step.output_slots)} output slots."
+            )
+        return list(implemented_step.output_slots)[0]
+
+    @staticmethod
+    def _extract_implementable_steps(schema: str) -> list[Step]:
+        """Extracts all implementable steps from the pipeline schema.
+
+        This method recursively traverses the pipeline schema specified in the script
+        to dynamically build a list of all implementable steps.
+        """
+
+        def _process_step(node: Step) -> None:
+            """Adds `step` to the `implementable_steps` list if it is implementable."""
+            if isinstance(node, IOStep):
+                return
+            elif isinstance(node, TemplatedStep):
+                _process_step(node.template_step)
+                return
+            elif isinstance(node, EmbarrassinglyParallelStep):
+                _process_step(node.step)
+                return
+            elif isinstance(node, ChoiceStep):
+                for choice_step in node.choices.values():
+                    _process_step(cast(Step, choice_step["step"]))
+                return
+            elif isinstance(node, HierarchicalStep):
+                implementable_steps.append(node)
+                for sub_step in node.nodes:
+                    _process_step(sub_step)
+                return
+            else:  # base Step
+                implementable_steps.append(node)
+                return
+
+        schema_steps = ALLOWED_SCHEMA_PARAMS[schema][0]
+
+        implementable_steps: list[Step] = []
+        for schema_step in schema_steps:
+            _process_step(schema_step)
+
+        return implementable_steps
+
+    @staticmethod
+    def _extract_pipeline_schema(script_path: Path) -> str:
+        """Extracts the relevant pipeline schema name.
+
+        The expectation is that the output slot's name is specified within the script
+        as a comment of the format:
+
+        .. code-block:: python
+            # PIPELINE_SCHEMA: development
+
+        If no pipeline schema is specified, "main" will be used by default.
+        """
+        schema = _extract_metadata("PIPELINE_SCHEMA", script_path)
+        return "main" if len(schema) == 0 else schema[0]
+
+    @staticmethod
+    def _write_metadata(info: dict[str, dict[str, str]]) -> None:
+        """Writes the implementation metadata to disk.
+
+        Parameters
+        ----------
+        info
+            The implementation metadata to write to disk.
+        """
+        with open(IMPLEMENTATION_METADATA, "w") as f:
+            yaml.dump(info, f, sort_keys=False)
+
+
+class PythonRecipe:
+    """A singularity recipe generator specific to implementations written in Python."""
+
+    BASE_IMAGE = (
+        "python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899"
+    )
+
+    def __init__(self, script_path: Path, recipe_path: Path, requirements: str) -> None:
+        self.script_path = script_path
+        self.recipe_path = recipe_path
+        self.requirements = requirements
+        self.text: str | None = None
+
+    def build(self) -> None:
+        """Builds the recipe for the container."""
+        logger.info(f"Building recipe for '{self.script_path.stem}'")
+
+        script_name = self.script_path.name
+        self.text = f"""
+Bootstrap: docker
+From: {self.BASE_IMAGE}
+
+%files
+    ./{script_name} /{script_name}
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install {self.requirements}
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /{script_name} '$@'"""
+
+    def write(self) -> None:
+        """Writes the recipe to disk.
+
+        Raises
+        ------
+        ValueError
+            If there is no recipe text to write a recipe from.
+        FileNotFoundError
+            If the recipe file was not written to disk.
+        """
+        logger.info(f"Writing recipe for '{self.script_path.stem}' to disk.")
+        if not self.text:
+            raise ValueError("No recipe text to build.")
+        if self.recipe_path.exists():
+            logger.warning(f"Recipe file {self.recipe_path} already exists. Overwriting it.")
+        with open(self.recipe_path, "w") as f:
+            f.write(self.text)
+            f.flush()
+        if not self.recipe_path.exists():
+            raise FileNotFoundError(f"Failed to write recipe to {self.recipe_path}.")
+
+
+####################
+# Helper functions #
+####################
+
+
+def _extract_metadata(key: str, script_path: Path) -> list[str]:
+    """Extracts the container metadata from the script comments.
+
+    Parameters
+    ----------
+    key
+        The key to search for in the script comments, e.g. "REQUIREMENTS" or "STEP_NAME".
+    script_path
+        The path to the script file.
+
+    Returns
+    -------
+        A list of metadata values found in the script comments.
+
+    Raises
+    ------
+    ValueError
+        If a key is found multiple times in the script.
+    """
+    metadata = []
+    for line in script_path.read_text().splitlines():
+        if key in line:
+            packed_line = line.replace(" ", "")
+            if packed_line.startswith(f"#{key}:"):
+                info = line.split(":")[1].strip()
+                metadata.append(info)
+
+    if len(metadata) > 1:
+        raise ValueError(
+            f"Found multiple {key.lower()} requests in {script_path}: {metadata}"
+            f"Please ensure the script contains only one comment of the form '# {key}: <request>'"
+        )
+    return metadata

easylink/implementation_metadata.yaml

@@ -1,195 +1,194 @@
-# development dummies
 step_1_python_pandas:
-  steps: 
-    - step_1
+  steps:
+  - step_1
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
 step_1a_python_pandas:
-  steps: 
-    - step_1a
+  steps:
+  - step_1a
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_1a_main_output: result.parquet
 step_1b_python_pandas:
-  steps: 
-    - step_1b
+  steps:
+  - step_1b
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_1b_main_output: result.parquet
 step_2_python_pandas:
-  steps: 
-    - step_2
+  steps:
+  - step_2
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
 step_3_python_pandas:
-  steps: 
-    - step_3
+  steps:
+  - step_3
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
     step_3_main_output: result.parquet
 step_4_python_pandas:
-  steps: 
-    - step_4
+  steps:
+  - step_4
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4_main_output: result.parquet
 step_5_python_pandas:
-  steps: 
-    - step_5
+  steps:
+  - step_5
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_5_main_output: result.parquet
 step_6_python_pandas:
-  steps: 
-    - step_6
+  steps:
+  - step_6
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_6_main_output: result.parquet
 step_4a_python_pandas:
-  steps: 
-    - step_4a
+  steps:
+  - step_4a
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4a_main_output: result.parquet
 step_4b_python_pandas:
-  steps: 
-    - step_4b
+  steps:
+  - step_4b
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4b_main_output: result.parquet
 step_4b_r:
-  steps: 
-    - step_4b
+  steps:
+  - step_4b
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
   script_cmd: Rscript /dummy_step.R
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4b_main_output: result.parquet
 step_1_python_pyspark:
-  steps: 
-    - step_1
+  steps:
+  - step_1
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_1_main_output: result.parquet
   requires_spark: true
 step_2_python_pyspark:
-  steps: 
-    - step_2
+  steps:
+  - step_2
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
   script_cmd: python3 /code/dummy_step.py
   outputs:
-    step_2_main_output: result.parquet 
+    step_2_main_output: result.parquet
   requires_spark: true
 step_3_python_pyspark:
-  steps: 
-    - step_3
+  steps:
+  - step_3
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
   script_cmd: python3 /code/dummy_step.py
   outputs:
     step_3_main_output: result.parquet
   requires_spark: true
 step_4_python_pyspark:
-  steps: 
-    - step_4
+  steps:
+  - step_4
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pyspark.sif
-  script_cmd: python3 /code/dummy_step.py 
+  script_cmd: python3 /code/dummy_step.py
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4_main_output: result.parquet
 step_1_r:
-  steps: 
-    - step_1
+  steps:
+  - step_1
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_1_main_output: result.parquet
   requires_spark: false
 step_2_r:
-  steps: 
-    - step_2
+  steps:
+  - step_2
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_2_main_output: result.parquet
   requires_spark: false
 step_3_r:
-  steps: 
-    - step_3
+  steps:
+  - step_3
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
   script_cmd: Rscript /dummy_step.R
   outputs:
     step_3_main_output: result.parquet
   requires_spark: false
 step_4_r:
-  steps: 
-    - step_4
+  steps:
+  - step_4
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
   script_cmd: Rscript /dummy_step.R
   env:
-    INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS
   outputs:
     step_4_main_output: result.parquet
   requires_spark: false
 step_1_and_step_2_combined_python_pandas:
   steps:
-    - step_1
-    - step_2
+  - step_1
+  - step_2
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
     step_2_main_output: result.parquet
 step_1_and_step_2_parallel_python_pandas:
   steps:
-    - step_1
-    - step_2
+  - step_1
+  - step_2
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   env:
-    INPUT_ENV_VARS: "STEP_1_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,STEP_2_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS"
+    INPUT_ENV_VARS: STEP_1_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,STEP_2_DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS
   outputs:
     step_2_main_output: result.parquet
 step_3_and_step_4_combined_python_pandas:
   steps:
-    - step_3
-    - step_4
+  - step_3
+  - step_4
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
     step_4_main_output: result.parquet
 step_1a_and_step_1b_combined_python_pandas:
   steps:
-    - step_1a
-    - step_1b
+  - step_1a
+  - step_1b
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
   script_cmd: python /dummy_step.py
   outputs:
-    step_1_main_output: result.parquet
+    step_1_main_output: result.parquet

easylink/pipeline_schema_constants/development.py

@@ -208,6 +208,14 @@ NODES = [
             "complex": {
                 "step": HierarchicalStep(
                     step_name="step_5_and_6",
+                    input_slots=[
+                        InputSlot(
+                            name="step_5_and_6_main_input",
+                            env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
+                            validator=validate_input_file_dummy,
+                        ),
+                    ],
+                    output_slots=[OutputSlot("step_5_and_6_main_output")],
                     nodes=[
                         Step(
                             step_name="step_5",
@@ -240,19 +248,33 @@ NODES = [
                             input_slot="step_6_main_input",
                         ),
                     ],
+                    input_slot_mappings=[
+                        InputSlotMapping(
+                            parent_slot="step_5_and_6_main_input",
+                            child_node="step_5",
+                            child_slot="step_5_main_input",
+                        ),
+                    ],
+                    output_slot_mappings=[
+                        OutputSlotMapping(
+                            parent_slot="step_5_and_6_main_output",
+                            child_node="step_6",
+                            child_slot="step_6_main_output",
+                        ),
+                    ],
                 ),
                 "input_slot_mappings": [
                     InputSlotMapping(
                         parent_slot="choice_section_main_input",
-                        child_node="step_5",
-                        child_slot="step_5_main_input",
+                        child_node="step_5_and_6",
+                        child_slot="step_5_and_6_main_input",
                     ),
                 ],
                 "output_slot_mappings": [
                     OutputSlotMapping(
                         parent_slot="choice_section_main_output",
-                        child_node="step_6",
-                        child_slot="step_6_main_output",
+                        child_node="step_5_and_6",
+                        child_slot="step_5_and_6_main_output",
                     ),
                 ],
             },

easylink/utilities/spark.smk

@@ -70,7 +70,7 @@ rule wait_for_spark_master:
         while true; do
 
             if [[ -e {params.spark_master_log_file} ]]; then
-                found=`grep -o "\(spark://.*$\)" {params.spark_master_log_file} || true`
+                found=`grep -o "\\(spark://.*$\\)" {params.spark_master_log_file} || true`
 
                 if [[ ! -z $found ]]; then
                     echo "Spark master URL found: $found"
@@ -178,7 +178,7 @@ rule wait_for_spark_worker:
         while true; do
 
             if [[ -e {params.spark_worker_log_file} ]]; then
-                found=`grep -o "\(Worker: Successfully registered with master $MASTER_URL\)" {params.spark_worker_log_file} || true`
+                found=`grep -o "\\(Worker: Successfully registered with master $MASTER_URL\\)" {params.spark_worker_log_file} || true`
 
                 if [[ ! -z $found ]]; then
                     echo "Spark Worker {wildcards.scatteritem} registered successfully"

{easylink-0.1.14.dist-info → easylink-0.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.14
+Version: 0.1.16
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers
@@ -21,6 +21,7 @@ Requires-Dist: snakemake-interface-executor-plugins<9.0.0
 Requires-Dist: snakemake-executor-plugin-slurm
 Requires-Dist: pandas-stubs
 Requires-Dist: pyarrow-stubs
+Requires-Dist: types-PyYAML
 Provides-Extra: docs
 Requires-Dist: sphinx<8.2.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
@@ -78,15 +79,16 @@ There are a few things to install in order to use this package:
 - Install singularity. 
 
   You may need to request it from your system admin. 
-  Refer to https://docs.sylabs.io/guides/4.1/admin-guide/installation.html. 
-  You can check if you already have singularity installed by running the command ``singularity --version``. For an
-  existing installation, your singularity version number is printed.
+  Refer to https://docs.sylabs.io/guides/4.1/admin-guide/installation.html.
+  You can check if you already have singularity installed by running the command
+  ``singularity --version``. For an existing installation, your singularity version
+  number is printed.
 
 - Install conda. 
   
-  We recommend `miniforge <https://github.com/conda-forge/miniforge>`_. You can check if you already
-  have conda installed by running the command ``conda --version``. For an existing installation, a version 
-  will be displayed.
+  We recommend `miniforge <https://github.com/conda-forge/miniforge>`_. You can
+  check if you already have conda installed by running the command ``conda --version``.
+  For an existing installation, a version will be displayed.
 
 - Install easylink, python and graphviz in a conda environment.
 

{easylink-0.1.14.dist-info → easylink-0.1.16.dist-info}/RECORD

@@ -1,21 +1,22 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=PIBqEOI-nqKFL9oJAWQQwlHuujG9Cd7EmdxDrThNQto,23
-easylink/cli.py,sha256=ARSKAljepNOEYd1VCS_QqBJQIBLzE3IgKiOb5-OROdY,6380
+easylink/_version.py,sha256=yF88-8vL8keLe6gCTumymw0UoMkWkSrJnzLru4zBCLQ,23
+easylink/cli.py,sha256=mv9l9XHojfhDK4hpDeV1E4iensgt6zx2ovkGBQ8x9xk,9745
 easylink/configuration.py,sha256=lfm8ViUpr1-O-EovTjKZbAlIht2EBv3RndN1mzYbmDE,12565
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
 easylink/implementation.py,sha256=4u3QgLOrNttfU9Kd_9u_lg3in4ePoYUfO9u_udwiuh0,10878
-easylink/implementation_metadata.yaml,sha256=VvlEu3Dvlmeh1MpzeYx91j22GiV-9mu3hZP5yVuW04o,6763
+easylink/implementation_metadata.yaml,sha256=_maN5UWFZxDykYcUrDXoEKMej4jeF_rZLt3QZj72kQM,6645
 easylink/pipeline.py,sha256=5KOYH5HyJjVlFoBRKGLs2hn5mpC3tPYG_ux3T1qSV9k,17504
 easylink/pipeline_graph.py,sha256=9ysX4wAkA-WkUoo15jSLAErncybE4tJwznVx7N_kwIA,23922
 easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
 easylink/rule.py,sha256=uoPj7yFFqiwvxlnhoejrZuPR3YX--y1k02uDDz3viTc,16196
 easylink/runner.py,sha256=cbCo5_NvvulmjjAaBCG6qCmbtJiHK-7NuDvbngdU_PY,6675
 easylink/step.py,sha256=u1AMPrYGNVb3ZH6uB_U0dUeJvOeQ2MoVHdlC8k63AA8,85226
+easylink/devtools/implementation_creator.py,sha256=mkiQ9nhtQC3mhxcG8IyvejzSK0WSkwplCztPLXbpXXQ,16199
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
 easylink/pipeline_schema_constants/__init__.py,sha256=HbN-NytoGuk8aTfe0Wal232UnLopFBQGe2uRjmg_igQ,1272
-easylink/pipeline_schema_constants/development.py,sha256=yRzkCiBqF_Jv3Y0GNvswVAWeZfKJRXk8Y8Q9ZhwCg_A,11596
+easylink/pipeline_schema_constants/development.py,sha256=XxcYYZDZM4IADp3eFPQCchD6-OtMp99GiyZBfSswzFo,12640
 easylink/pipeline_schema_constants/testing.py,sha256=8vVGj7opZ9Uzj7EHGMbgXyZj3_SboIeUPB0XlZkmvrM,18901
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
@@ -40,11 +41,11 @@ easylink/utilities/aggregator_utils.py,sha256=pqBog6kEX4MXBBMjQtHFlE5gEMqRWb5VFl
 easylink/utilities/data_utils.py,sha256=CcnM3u0_MQDQo3jMs3E4IK_rz8wAsFdJ674fZxYEFZg,4620
 easylink/utilities/general_utils.py,sha256=El1W0nn4P27sRBGotNQb-9du-Gbhk9ggSuu4vmGDfwo,4591
 easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,924
-easylink/utilities/spark.smk,sha256=tQ7RArNQzhjbaBQQcRORB4IxxkuDx4gPHUBcWHDYJ_U,5795
+easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=UOz4hjkEPqaAz0RrDkDYYej79lLSaq0VVVSH_tF1z0o,3838
 easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
-easylink-0.1.14.dist-info/METADATA,sha256=zJQ4KHmrAf6NoEW_MMbqj9RmpuBav6IggUTxnXFfezg,3449
-easylink-0.1.14.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
-easylink-0.1.14.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.14.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.14.dist-info/RECORD,,
+easylink-0.1.16.dist-info/METADATA,sha256=xkRlfeXuPHvvZXwEHaObnpu6MsOWSF6Lu-1wi7wRlJQ,3477
+easylink-0.1.16.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+easylink-0.1.16.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.16.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.16.dist-info/RECORD,,

{easylink-0.1.14.dist-info → easylink-0.1.16.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.1.0)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any