easylink 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.16"
+__version__ = "0.1.17"

easylink/implementation.py

@@ -135,8 +135,17 @@ class Implementation:
 
     @property
     def outputs(self) -> dict[str, list[str]]:
-        """The expected output metadata."""
-        return self._metadata["outputs"]
+        """The expected output paths. If output metadata is provided, use it. Otherwise,
+        assume that the output is a sub-directory with the name of the output slot.
+        If there is only one output slot, use '.'."""
+        if len(self.output_slots) == 1:
+            return self._metadata.get("outputs", {list(self.output_slots.keys())[0]: "."})
+        return {
+            output_slot_name: self._metadata.get("outputs", {}).get(
+                output_slot_name, output_slot_name
+            )
+            for output_slot_name in self.output_slots
+        }
 
 
 class NullImplementation:

easylink/implementation_metadata.yaml

@@ -192,3 +192,25 @@ step_1a_and_step_1b_combined_python_pandas:
   script_cmd: python /dummy_step.py
   outputs:
     step_1_main_output: result.parquet
+dummy_step_1_for_output_dir_example:
+  steps:
+  - step_1_for_output_dir_example
+  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
+  script_cmd: python /dummy_step_1_for_output_dir_example.py
+  outputs:
+    step_1_main_output_directory: output_dir/
+dummy_step_1_for_output_dir_example_default:
+  steps:
+  - step_1_for_output_dir_example
+  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
+  script_cmd: python /dummy_step_1_for_output_dir_example.py
+  # leave outputs out for testing purposes
+  # outputs:
+  #   step_1_main_output_directory: output_dir/
+dummy_step_2_for_output_dir_example:
+  steps:
+  - step_2_for_output_dir_example
+  image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_2_for_output_dir_example.sif
+  script_cmd: python /dummy_step_2_for_output_dir_example.py
+  outputs:
+    step_2_main_output: result.parquet

easylink/pipeline_schema_constants/__init__.py

@@ -17,6 +17,7 @@ ALLOWED_SCHEMA_PARAMS = {
 
 TESTING_SCHEMA_PARAMS = {
     "integration": testing.SCHEMA_PARAMS_ONE_STEP,
+    "output_dir": testing.SCHEMA_PARAMS_OUTPUT_DIR,
     "combine_bad_topology": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
     "combine_bad_implementation_names": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,
     "nested_templated_steps": testing.SCHEMA_PARAMS_NESTED_TEMPLATED_STEPS,

easylink/pipeline_schema_constants/testing.py

@@ -26,7 +26,7 @@ from easylink.step import (
 )
 from easylink.utilities.aggregator_utils import concatenate_datasets
 from easylink.utilities.splitter_utils import split_data_in_two
-from easylink.utilities.validation_utils import validate_input_file_dummy
+from easylink.utilities.validation_utils import validate_dir, validate_input_file_dummy
 
 NODES_ONE_STEP = [
     InputStep(),
@@ -582,3 +582,55 @@ EDGES_ONE_STEP_TWO_ISLOTS = [
     ),
 ]
 SCHEMA_PARAMS_EP_HIERARCHICAL_STEP = (NODES_EP_HIERARCHICAL_STEP, EDGES_ONE_STEP_TWO_ISLOTS)
+
+NODES_OUTPUT_DIR = [
+    InputStep(),
+    Step(
+        step_name="step_1_for_output_dir_example",
+        input_slots=[
+            InputSlot(
+                name="step_1_main_input",
+                env_var="STEP_1_MAIN_INPUT_FILE_PATHS",
+                validator=validate_input_file_dummy,
+            )
+        ],
+        output_slots=[OutputSlot("step_1_main_output_directory")],
+    ),
+    Step(
+        step_name="step_2_for_output_dir_example",
+        input_slots=[
+            InputSlot(
+                name="step_2_main_input",
+                env_var="DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH",
+                validator=validate_dir,
+            )
+        ],
+        output_slots=[OutputSlot("step_2_main_output")],
+    ),
+    OutputStep(
+        input_slots=[
+            InputSlot(name="result", env_var=None, validator=validate_input_file_dummy)
+        ],
+    ),
+]
+EDGES_OUTPUT_DIR = [
+    EdgeParams(
+        source_node="input_data",
+        target_node="step_1_for_output_dir_example",
+        output_slot="all",
+        input_slot="step_1_main_input",
+    ),
+    EdgeParams(
+        source_node="step_1_for_output_dir_example",
+        target_node="step_2_for_output_dir_example",
+        output_slot="step_1_main_output_directory",
+        input_slot="step_2_main_input",
+    ),
+    EdgeParams(
+        source_node="step_2_for_output_dir_example",
+        target_node="results",
+        output_slot="step_2_main_output",
+        input_slot="result",
+    ),
+]
+SCHEMA_PARAMS_OUTPUT_DIR = (NODES_OUTPUT_DIR, EDGES_OUTPUT_DIR)

easylink/rule.py

@@ -17,6 +17,7 @@ import os
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from dataclasses import dataclass
+from pathlib import Path
 
 
 class Rule(ABC):
@@ -125,6 +126,18 @@ class ImplementedRule(Rule):
     def _build_io(self) -> str:
         """Builds the input/output portion of the rule."""
         log_path_chunk_adder = "-{chunk}" if self.is_embarrassingly_parallel else ""
+        # Handle output files vs directories
+        files = [path for path in self.output if Path(path).suffix != ""]
+        if len(files) == len(self.output):
+            output = self.output
+        elif len(files) == 0:
+            if len(self.output) != 1:
+                raise NotImplementedError("Multiple output directories is not supported.")
+            output = f"directory('{self.output[0]}')"
+        else:
+            raise NotImplementedError(
+                "Mixed output types (files and directories) is not supported."
+            )
         io_str = (
             f"""
 rule:
@@ -132,7 +145,7 @@ rule:
     message: "Running {self.step_name} implementation: {self.implementation_name}" """
             + self._build_input()
             + f"""        
-    output: {self.output}
+    output: {output}
     log: "{self.diagnostics_dir}/{self.name}-output{log_path_chunk_adder}.log"
     container: "{self.image_path}" """
         )

easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./dummy_step_1_for_output_dir_example.py /dummy_step_1_for_output_dir_example.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /dummy_step_1_for_output_dir_example.py '$@'

easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py

@@ -0,0 +1,18 @@
+# PIPELINE_SCHEMA: output_dir
+# STEP_NAME: step_1_for_output_dir_example
+# REQUIREMENTS: pandas==2.1.2 pyarrow
+
+import os
+from pathlib import Path
+
+import pandas as pd
+
+data = pd.read_parquet(os.environ["STEP_1_MAIN_INPUT_FILE_PATHS"])
+
+print(data)
+
+dir_path = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+dir_path.mkdir(parents=True, exist_ok=True)
+
+for i in range(3):
+    data.to_parquet(dir_path / f"result_{i}.parquet")

easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./dummy_step_2_for_output_dir_example.py /dummy_step_2_for_output_dir_example.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /dummy_step_2_for_output_dir_example.py '$@'

easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py

@@ -0,0 +1,22 @@
+# PIPELINE_SCHEMA: output_dir
+# STEP_NAME: step_2_for_output_dir_example
+# REQUIREMENTS: pandas==2.1.2 pyarrow
+
+import os
+import shutil
+from pathlib import Path
+
+import pandas as pd
+
+dir_path = Path(os.environ["DUMMY_CONTAINER_MAIN_INPUT_DIR_PATH"])
+saved = False
+
+for i, f in enumerate([f for f in dir_path.iterdir() if f.is_file()]):
+    if "snakemake" in str(f):
+        continue
+
+    if not saved:
+        shutil.copy(f, os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+        saved = True
+
+    print(pd.read_parquet(f))

easylink/utilities/validation_utils.py

@@ -50,3 +50,9 @@ def validate_input_file_dummy(filepath: str) -> None:
         raise LookupError(
             f"Data file {filepath} is missing required column(s) {missing_columns}"
         )
+
+
+def validate_dir(filepath: str) -> None:
+    input_path = Path(filepath)
+    if not input_path.is_dir():
+        raise NotADirectoryError(f"The path {filepath} is not a directory.")

{easylink-0.1.16.dist-info → easylink-0.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.16
+Version: 0.1.17
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.16.dist-info → easylink-0.1.17.dist-info}/RECORD

@@ -1,23 +1,23 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=yF88-8vL8keLe6gCTumymw0UoMkWkSrJnzLru4zBCLQ,23
+easylink/_version.py,sha256=BzIjnki8Bz3evNWo6bjGxxpLhy_tN9MRYhtM0MnDiWs,23
 easylink/cli.py,sha256=mv9l9XHojfhDK4hpDeV1E4iensgt6zx2ovkGBQ8x9xk,9745
 easylink/configuration.py,sha256=lfm8ViUpr1-O-EovTjKZbAlIht2EBv3RndN1mzYbmDE,12565
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
-easylink/implementation.py,sha256=4u3QgLOrNttfU9Kd_9u_lg3in4ePoYUfO9u_udwiuh0,10878
-easylink/implementation_metadata.yaml,sha256=_maN5UWFZxDykYcUrDXoEKMej4jeF_rZLt3QZj72kQM,6645
+easylink/implementation.py,sha256=XLSat6_IXFn-nH6X8AazmfWhDtTK4GtA7yiht9QLlQQ,11366
+easylink/implementation_metadata.yaml,sha256=0BQ_NIV29EtJ6G_wGD_-2OYPjYw2sNLAgCg2GbNwcuI,7662
 easylink/pipeline.py,sha256=5KOYH5HyJjVlFoBRKGLs2hn5mpC3tPYG_ux3T1qSV9k,17504
 easylink/pipeline_graph.py,sha256=9ysX4wAkA-WkUoo15jSLAErncybE4tJwznVx7N_kwIA,23922
 easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
-easylink/rule.py,sha256=uoPj7yFFqiwvxlnhoejrZuPR3YX--y1k02uDDz3viTc,16196
+easylink/rule.py,sha256=NusEUtBxx18L7UCcgDi3KKooFxSUgyS4eisVM5aPqFE,16770
 easylink/runner.py,sha256=cbCo5_NvvulmjjAaBCG6qCmbtJiHK-7NuDvbngdU_PY,6675
 easylink/step.py,sha256=u1AMPrYGNVb3ZH6uB_U0dUeJvOeQ2MoVHdlC8k63AA8,85226
 easylink/devtools/implementation_creator.py,sha256=mkiQ9nhtQC3mhxcG8IyvejzSK0WSkwplCztPLXbpXXQ,16199
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=HbN-NytoGuk8aTfe0Wal232UnLopFBQGe2uRjmg_igQ,1272
+easylink/pipeline_schema_constants/__init__.py,sha256=FUngnh80yfpr76j18iBVKSoR8-5zxQj_mW_muUMrafw,1324
 easylink/pipeline_schema_constants/development.py,sha256=XxcYYZDZM4IADp3eFPQCchD6-OtMp99GiyZBfSswzFo,12640
-easylink/pipeline_schema_constants/testing.py,sha256=8vVGj7opZ9Uzj7EHGMbgXyZj3_SboIeUPB0XlZkmvrM,18901
+easylink/pipeline_schema_constants/testing.py,sha256=UDmVVjI1SiDktMbJ2CrSb7amHSYNwhgqNkXhl4lYxQw,20459
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
 easylink/steps/dev/build-containers-local.sh,sha256=Wy3pfcyt7I-BNvHcr7ZXDe0g5Ihd00BIPqt9YuRbLeA,259
 easylink/steps/dev/build-containers-remote.sh,sha256=Hy-kaaXf-ta6n8SzOz_ahByjMY5T7J71MvzXRXDvQw8,271
@@ -36,6 +36,10 @@ easylink/steps/dev/python_pyspark/python_pyspark.def,sha256=j_RmVjspmXGOhJTr10ED
 easylink/steps/dev/r/README.md,sha256=dPjZdDTqcJsZCiwhddzlOj1ob0P7YocZUNFrLIGM1-0,1201
 easylink/steps/dev/r/dummy_step.R,sha256=1TWZY8CEkT6gavrulBxFsKbDSKJJjk0NtJrGH7TIikE,4975
 easylink/steps/dev/r/r-image.def,sha256=LrhXlt0C3k7d_VJWopRPEVARnFWSuq_oILlwo7g03bE,627
+easylink/steps/output_dir/dummy_step_1_for_output_dir_example.def,sha256=CkQVG-uDRQ9spAavdkZbhx2GD_fRsKZGELPrr8yltsc,550
+easylink/steps/output_dir/dummy_step_1_for_output_dir_example.py,sha256=dI0OWugE35ABLcSwsI-T3C4dvuPTKXwjE52dtSsCo8Y,428
+easylink/steps/output_dir/dummy_step_2_for_output_dir_example.def,sha256=9gShg1EDJEHZcz7Z5VfZ1A4Gpm9XQes8ezn6rAZDgDM,550
+easylink/steps/output_dir/dummy_step_2_for_output_dir_example.py,sha256=DMJW5TXjhELxhY4U9q2RpLjqxlS1YSosTGL2AfRnaZM,521
 easylink/utilities/__init__.py,sha256=0U33kbv4hoMfFQ_lh5hLwifxRPzOgkLkjKLYxmaK10g,196
 easylink/utilities/aggregator_utils.py,sha256=pqBog6kEX4MXBBMjQtHFlE5gEMqRWb5VFl64u0Lr__g,972
 easylink/utilities/data_utils.py,sha256=CcnM3u0_MQDQo3jMs3E4IK_rz8wAsFdJ674fZxYEFZg,4620
@@ -43,9 +47,9 @@ easylink/utilities/general_utils.py,sha256=El1W0nn4P27sRBGotNQb-9du-Gbhk9ggSuu4v
 easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,924
 easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=UOz4hjkEPqaAz0RrDkDYYej79lLSaq0VVVSH_tF1z0o,3838
-easylink/utilities/validation_utils.py,sha256=W9r_RXcivJjfpioLhONirfwdByYttxNsVY489_sbrYQ,1683
-easylink-0.1.16.dist-info/METADATA,sha256=xkRlfeXuPHvvZXwEHaObnpu6MsOWSF6Lu-1wi7wRlJQ,3477
-easylink-0.1.16.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-easylink-0.1.16.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.16.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.16.dist-info/RECORD,,
+easylink/utilities/validation_utils.py,sha256=rOIeQbbXXdsuL2hI0i2gApAWfiNJXMwYH4pmw8uLrGM,1867
+easylink-0.1.17.dist-info/METADATA,sha256=fl6OzaU74KHClV-dADXheUoKBMuIs8pUqLANelfaqBY,3477
+easylink-0.1.17.dist-info/WHEEL,sha256=A8Eltl-h0W-qZDVezsLjjslosEH_pdYC2lQ0JcbgCzs,91
+easylink-0.1.17.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.17.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.17.dist-info/RECORD,,

{easylink-0.1.16.dist-info → easylink-0.1.17.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.7.0)
 Root-Is-Purelib: true
 Tag: py3-none-any