easylink 0.1.17__py3-none-any.whl → 0.1.18__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.17"
+__version__ = "0.1.18"

easylink/cli.py

@@ -91,6 +91,11 @@ SHARED_OPTIONS = [
         default=False,
         help="Do not save the results in a timestamped sub-directory of ``--output-dir``.",
     ),
+    click.option(
+        "--schema",
+        hidden=True,
+        default="main",
+    ),
 ]
 
 VERBOSE_WITH_DEBUGGER_OPTIONS = [
@@ -165,6 +170,7 @@ def run(
     input_data: str,
     output_dir: str | None,
     no_timestamp: bool,
+    schema: str,
     computing_environment: str | None,
     verbose: int,
     with_debugger: bool,
@@ -190,6 +196,7 @@ def run(
         input_data=input_data,
         computing_environment=computing_environment,
         results_dir=results_dir,
+        schema_name=schema,
     )
     logger.info("*** FINISHED ***")
 
@@ -201,6 +208,7 @@ def generate_dag(
     input_data: str,
     output_dir: str | None,
     no_timestamp: bool,
+    schema: str,
     verbose: int,
     with_debugger: bool,
 ) -> None:
@@ -223,6 +231,7 @@ def generate_dag(
         input_data=input_data,
         computing_environment=None,
         results_dir=results_dir,
+        schema_name=schema,
     )
     logger.info("*** DAG saved to result directory ***")
 

easylink/configuration.py

@@ -14,7 +14,7 @@ from typing import Any
 
 from layered_config_tree import LayeredConfigTree
 
-from easylink.pipeline_schema import PIPELINE_SCHEMAS, PipelineSchema
+from easylink.pipeline_schema import PipelineSchema
 from easylink.utilities.data_utils import load_yaml
 from easylink.utilities.general_utils import exit_with_validation_error
 
@@ -67,9 +67,8 @@ class Config(LayeredConfigTree):
         A dictionary of all specifications required to run the pipeline. This
         includes the pipeline, input data, and computing environment specifications,
         as well as the results directory.
-    potential_schemas
-        A list of potential schemas to validate the pipeline configuration against.
-        This is primarily used for testing purposes. Defaults to the supported schemas.
+    schema_name
+        The name of the schema to validate the pipeline configuration against.
 
     Attributes
     ----------
@@ -82,22 +81,14 @@ class Config(LayeredConfigTree):
     input_data
         The input data filepaths.
     schema
-        The :class:`~easylink.pipeline_schema.PipelineSchema` that successfully
-        validated the requested pipeline.
-
-    Notes
-    -----
-    The requested pipeline is checked against a set of supported
-    ``PipelineSchemas``. The first schema that successfully validates is assumed
-    to be the correct one and is attached to the ``Config`` object and its
-    :meth:`~easylink.pipeline_schema.PipelineSchema.configure_pipeline`
-    method is called.
+        The :class:`~easylink.pipeline_schema.PipelineSchema`.
+
     """
 
     def __init__(
         self,
         config_params: dict[str, Any],
-        potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
+        schema_name: str = "main",
     ) -> None:
         super().__init__(layers=["initial_data", "default", "user_configured"])
         self.update(DEFAULT_ENVIRONMENT, layer="default")
@@ -108,9 +99,7 @@ class Config(LayeredConfigTree):
             # Set slurm defaults to empty dict instead of None so that we don't get errors
             # in slurm_resources property
             self.update({"environment": {"slurm": {}}}, layer="default")
-        if not isinstance(potential_schemas, list):
-            potential_schemas = [potential_schemas]
-        self.update({"schema": self._get_schema(potential_schemas)}, layer="initial_data")
+        self.update({"schema": self._get_schema(schema_name)}, layer="initial_data")
         self.schema.configure_pipeline(self.pipeline, self.input_data)
         self._validate()
         self.freeze()
@@ -173,22 +162,22 @@ class Config(LayeredConfigTree):
     # Setup Methods #
     #################
 
-    def _get_schema(self, potential_schemas: list[PipelineSchema]) -> PipelineSchema:
+    def _get_schema(self, schema_name: str = "main") -> PipelineSchema:
         """Returns the first :class:`~easylink.pipeline_schema.PipelineSchema` that validates the requested pipeline.
 
         Parameters
         ----------
-        potential_schemas
-            ``PipelineSchemas`` to validate the pipeline configuration against.
+        schema_name
+            The name of the specific ``PipelineSchema`` to validate the pipeline configuration against.
 
         Returns
         -------
-            The first ``PipelineSchema`` that validates the requested pipeline configuration.
+            The requested ``PipelineSchema`` if it validates the requested pipeline configuration.
 
         Raises
         ------
         SystemExit
-            If the pipeline configuration is not valid for any of the ``potential_schemas``,
+            If the pipeline configuration is not valid for the requested schema,
             the program exits with a non-zero code and all validation errors found
             are logged.
 
@@ -197,20 +186,15 @@ class Config(LayeredConfigTree):
         This acts as the pipeline configuration file's validation method since
         we can only find a matching ``PipelineSchema`` if that file is valid.
 
-        This method returns the *first* ``PipelineSchema`` that validates and does
-        not attempt to check additional ones.
         """
         errors = defaultdict(dict)
         # Try each schema until one is validated
-        for schema in potential_schemas:
-            logs = schema.validate_step(self.pipeline, self.input_data)
-            if logs:
-                errors[PIPELINE_ERRORS_KEY][schema.name] = logs
-                pass  # try the next schema
-            else:  # schema was validated
-                return schema
-        # No schemas were validated
-        exit_with_validation_error(dict(errors))
+        schema = PipelineSchema.get_schema(schema_name)
+        logs = schema.validate_step(self.pipeline, self.input_data)
+        if logs:
+            errors[PIPELINE_ERRORS_KEY][schema.name] = logs
+            exit_with_validation_error(dict(errors))
+        return schema
 
     def _validate(self) -> None:
         """Validates the ``Config``.

easylink/devtools/implementation_creator.py

@@ -19,7 +19,7 @@ from typing import cast
 import yaml
 from loguru import logger
 
-from easylink.pipeline_schema_constants import ALLOWED_SCHEMA_PARAMS
+from easylink.pipeline_schema_constants import SCHEMA_PARAMS
 from easylink.step import (
     ChoiceStep,
     EmbarrassinglyParallelStep,
@@ -244,17 +244,17 @@ class ImplementationCreator:
     @staticmethod
     def _extract_output_slot(script_path: Path, step_name: str) -> str:
         """Extracts the name of the output slot that this script is implementing."""
-        schema = ImplementationCreator._extract_pipeline_schema(script_path)
-        implementable_steps = ImplementationCreator._extract_implementable_steps(schema)
+        schema_name = ImplementationCreator._extract_pipeline_schema_name(script_path)
+        implementable_steps = ImplementationCreator._extract_implementable_steps(schema_name)
         step_names = [step.name for step in implementable_steps]
         if step_name not in step_names:
             raise ValueError(
-                f"'{step_name}' does not exist as an implementable step in the '{schema}' pipeline schema. "
+                f"'{step_name}' does not exist as an implementable step in the '{schema_name}' pipeline schema. "
             )
         duplicates = list(set([step for step in step_names if step_names.count(step) > 1]))
         if duplicates:
             raise ValueError(
-                f"Multiple implementable steps with the same name found in the '{schema}' "
+                f"Multiple implementable steps with the same name found in the '{schema_name}' "
                 f"pipeline schema: {duplicates}."
             )
         implemented_step = [step for step in implementable_steps if step.name == step_name][0]
@@ -266,7 +266,7 @@ class ImplementationCreator:
         return list(implemented_step.output_slots)[0]
 
     @staticmethod
-    def _extract_implementable_steps(schema: str) -> list[Step]:
+    def _extract_implementable_steps(schema_name: str) -> list[Step]:
         """Extracts all implementable steps from the pipeline schema.
 
         This method recursively traverses the pipeline schema specified in the script
@@ -296,8 +296,7 @@ class ImplementationCreator:
                 implementable_steps.append(node)
                 return
 
-        schema_steps = ALLOWED_SCHEMA_PARAMS[schema][0]
-
+        schema_steps, _edges = SCHEMA_PARAMS[schema_name]
         implementable_steps: list[Step] = []
         for schema_step in schema_steps:
             _process_step(schema_step)
@@ -305,7 +304,7 @@ class ImplementationCreator:
         return implementable_steps
 
     @staticmethod
-    def _extract_pipeline_schema(script_path: Path) -> str:
+    def _extract_pipeline_schema_name(script_path: Path) -> str:
         """Extracts the relevant pipeline schema name.
 
         The expectation is that the output slot's name is specified within the script
@@ -316,8 +315,11 @@ class ImplementationCreator:
 
         If no pipeline schema is specified, "main" will be used by default.
         """
-        schema = _extract_metadata("PIPELINE_SCHEMA", script_path)
-        return "main" if len(schema) == 0 else schema[0]
+        schema_name_list: list[str] = _extract_metadata("PIPELINE_SCHEMA", script_path)
+        schema_name = "main" if len(schema_name_list) == 0 else schema_name_list[0]
+        if schema_name not in SCHEMA_PARAMS:
+            raise ValueError(f"Pipeline schema '{schema_name}' is not supported.")
+        return schema_name
 
     @staticmethod
     def _write_metadata(info: dict[str, dict[str, str]]) -> None:

easylink/implementation_metadata.yaml

@@ -204,9 +204,6 @@ dummy_step_1_for_output_dir_example_default:
   - step_1_for_output_dir_example
   image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/zmbc/dummy_step_1_for_output_dir_example.sif
   script_cmd: python /dummy_step_1_for_output_dir_example.py
-  # leave outputs out for testing purposes
-  # outputs:
-  #   step_1_main_output_directory: output_dir/
 dummy_step_2_for_output_dir_example:
   steps:
   - step_2_for_output_dir_example

easylink/pipeline_schema.py

@@ -14,7 +14,7 @@ from pathlib import Path
 from layered_config_tree import LayeredConfigTree
 
 from easylink.graph_components import EdgeParams, ImplementationGraph
-from easylink.pipeline_schema_constants import ALLOWED_SCHEMA_PARAMS
+from easylink.pipeline_schema_constants import SCHEMA_PARAMS
 from easylink.step import HierarchicalStep, NonLeafConfigurationState, Step
 
 
@@ -39,7 +39,7 @@ class PipelineSchema(HierarchicalStep):
 
     Notes
     -----
-    All ``PipelineSchema`` instances are intended to be created by the :meth:`_get_schemas`
+    A ``PipelineSchema`` is intended to be constructed by the :meth:`get_schema`
     class method.
 
     The ``PipelineSchema`` is a high-level abstraction; it represents the desired
@@ -159,22 +159,21 @@ class PipelineSchema(HierarchicalStep):
         )
 
     @classmethod
-    def _get_schemas(cls) -> list["PipelineSchema"]:
+    def get_schema(cls, name: str = "main") -> list["PipelineSchema"]:
         """Gets all allowable ``PipelineSchemas``.
 
         These ``PipelineSchemas`` represent the fully supported pipelines and are
         used to validate the user-requested pipeline.
 
+        Parameters
+        ----------
+        name
+            The name of the ``PipelineSchema`` to get.
+
         Returns
         -------
-            All allowable ``PipelineSchemas``.
+            The requested ``PipelineSchema``.
         """
-        return [
-            cls(name, nodes=nodes, edges=edges)
-            for name, (nodes, edges) in ALLOWED_SCHEMA_PARAMS.items()
-        ]
-
-
-PIPELINE_SCHEMAS = PipelineSchema._get_schemas()
-"""All allowable :class:`PipelineSchemas<PipelineSchema>` to validate the requested
-pipeline against."""
+        if name not in SCHEMA_PARAMS:
+            raise ValueError(f"Pipeline schema '{name}' is not supported.")
+        return cls(name, *SCHEMA_PARAMS[name])

easylink/pipeline_schema_constants/__init__.py

@@ -11,11 +11,10 @@ package defines the nodes and edges required to instantiate such ``PipelineSchem
 
 from easylink.pipeline_schema_constants import development, testing
 
-ALLOWED_SCHEMA_PARAMS = {
+SCHEMA_PARAMS = {
+    "main": "TODO",
+    # development and testing
     "development": development.SCHEMA_PARAMS,
-}
-
-TESTING_SCHEMA_PARAMS = {
     "integration": testing.SCHEMA_PARAMS_ONE_STEP,
     "output_dir": testing.SCHEMA_PARAMS_OUTPUT_DIR,
     "combine_bad_topology": testing.SCHEMA_PARAMS_BAD_COMBINED_TOPOLOGY,

easylink/runner.py

@@ -19,7 +19,6 @@ from snakemake.cli import main as snake_main
 
 from easylink.configuration import Config, load_params_from_specification
 from easylink.pipeline import Pipeline
-from easylink.pipeline_schema import PIPELINE_SCHEMAS, PipelineSchema
 from easylink.utilities.data_utils import (
     copy_configuration_files_to_results_directory,
     create_results_directory,
@@ -35,8 +34,8 @@ def main(
     input_data: str | Path,
     computing_environment: str | Path | None,
     results_dir: str | Path,
-    debug=False,
-    potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
+    schema_name: str = "main",
+    debug: bool = False,
 ) -> None:
     """Runs an EasyLink command.
 
@@ -60,17 +59,16 @@ def main(
         to run the pipeline on. If None, the pipeline will be run locally.
     results_dir
         The directory to write results and incidental files (logs, etc.) to.
+    schema_name
+        The name of the schema to validate the pipeline configuration against.
     debug
         If False (the default), will suppress some of the workflow output. This
         is intended to only be used for testing and development purposes.
-    potential_schemas
-        A list of potential schemas to validate the pipeline configuration against.
-        This is primarily used for testing purposes. Defaults to the supported schemas.
     """
     config_params = load_params_from_specification(
         pipeline_specification, input_data, computing_environment, results_dir
     )
-    config = Config(config_params, potential_schemas)
+    config = Config(config_params, schema_name)
     pipeline = Pipeline(config)
     # After validation is completed, create the results directory
     create_results_directory(Path(results_dir))

{easylink-0.1.17.dist-info → easylink-0.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.17
+Version: 0.1.18
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.17.dist-info → easylink-0.1.18.dist-info}/RECORD

@@ -1,21 +1,21 @@
 easylink/__about__.py,sha256=2-oxCfu9t9yUJouLDwqYRZ0eii8kN25SxRzsawjWjho,440
 easylink/__init__.py,sha256=gGMcIVfiVnHtlDw5mZwhevcDb2wt-kuP6F64gnkFack,159
-easylink/_version.py,sha256=BzIjnki8Bz3evNWo6bjGxxpLhy_tN9MRYhtM0MnDiWs,23
-easylink/cli.py,sha256=mv9l9XHojfhDK4hpDeV1E4iensgt6zx2ovkGBQ8x9xk,9745
-easylink/configuration.py,sha256=lfm8ViUpr1-O-EovTjKZbAlIht2EBv3RndN1mzYbmDE,12565
+easylink/_version.py,sha256=6BiuMUkhwQp6bzUZSF8np8F1NwCltEtK0sPBF__tepU,23
+easylink/cli.py,sha256=v8OALTAI3WlNELrHuGQumlJFdmYML4K-XX_OtqSJYZM,9925
+easylink/configuration.py,sha256=rFPTZMEBZjiXYZWesUqpppOj6ONxp3sybf5g9MiDsOY,11639
 easylink/graph_components.py,sha256=zZDZXg5smReHO3ryQC4pao24wyKXzWDe6jS3C6fM2ak,13892
 easylink/implementation.py,sha256=XLSat6_IXFn-nH6X8AazmfWhDtTK4GtA7yiht9QLlQQ,11366
-easylink/implementation_metadata.yaml,sha256=0BQ_NIV29EtJ6G_wGD_-2OYPjYw2sNLAgCg2GbNwcuI,7662
+easylink/implementation_metadata.yaml,sha256=trq5CvSSZRmqRQ979o68L2QONvlv-ncFXS-rh3-79Uk,7558
 easylink/pipeline.py,sha256=5KOYH5HyJjVlFoBRKGLs2hn5mpC3tPYG_ux3T1qSV9k,17504
 easylink/pipeline_graph.py,sha256=9ysX4wAkA-WkUoo15jSLAErncybE4tJwznVx7N_kwIA,23922
-easylink/pipeline_schema.py,sha256=Q2sCpsC-F2W0yxVP7ufunowDepOBrRVENXOdap9J5iY,6921
+easylink/pipeline_schema.py,sha256=FieJBa3rKgaCIB9QDuQEfWJ9joNBUUp6iHT6xmns-Vk,6886
 easylink/rule.py,sha256=NusEUtBxx18L7UCcgDi3KKooFxSUgyS4eisVM5aPqFE,16770
-easylink/runner.py,sha256=cbCo5_NvvulmjjAaBCG6qCmbtJiHK-7NuDvbngdU_PY,6675
+easylink/runner.py,sha256=GhkPGDh9UFOb38ksqXpMKZoxXs9hZaOFzZDo2jlEp-U,6458
 easylink/step.py,sha256=u1AMPrYGNVb3ZH6uB_U0dUeJvOeQ2MoVHdlC8k63AA8,85226
-easylink/devtools/implementation_creator.py,sha256=mkiQ9nhtQC3mhxcG8IyvejzSK0WSkwplCztPLXbpXXQ,16199
+easylink/devtools/implementation_creator.py,sha256=ddzJltlzOfvzwAMuInovCbfn3IM2u_s7I_dObWV4os0,16430
 easylink/images/spark_cluster/Dockerfile,sha256=3PHotbR4jdjVYRHOJ0VQW55b5Qd4tQ1pLLQMrTKWVA0,576
 easylink/images/spark_cluster/README.md,sha256=KdgSttZRplNNWqHn4K1GTsTIab3dTOSG4V99QPLxSp8,569
-easylink/pipeline_schema_constants/__init__.py,sha256=FUngnh80yfpr76j18iBVKSoR8-5zxQj_mW_muUMrafw,1324
+easylink/pipeline_schema_constants/__init__.py,sha256=45S-Q69CugGfBroHuGR8c7Jlq1wqAy5lRtys5C_0--M,1337
 easylink/pipeline_schema_constants/development.py,sha256=XxcYYZDZM4IADp3eFPQCchD6-OtMp99GiyZBfSswzFo,12640
 easylink/pipeline_schema_constants/testing.py,sha256=UDmVVjI1SiDktMbJ2CrSb7amHSYNwhgqNkXhl4lYxQw,20459
 easylink/steps/dev/README.md,sha256=u9dZUggpY2Lf2qb-xkDLWWgHjcmi4osbQtzSNo4uklE,4549
@@ -48,8 +48,8 @@ easylink/utilities/paths.py,sha256=KM1GlnsAcKbUJrC4LZKpeJfPljxe_aXP1ZhVp43TYRA,9
 easylink/utilities/spark.smk,sha256=kGtpem7LfQc71tMh5WAYaqKnHQKFvcdhPQSdumOP70k,5799
 easylink/utilities/splitter_utils.py,sha256=UOz4hjkEPqaAz0RrDkDYYej79lLSaq0VVVSH_tF1z0o,3838
 easylink/utilities/validation_utils.py,sha256=rOIeQbbXXdsuL2hI0i2gApAWfiNJXMwYH4pmw8uLrGM,1867
-easylink-0.1.17.dist-info/METADATA,sha256=fl6OzaU74KHClV-dADXheUoKBMuIs8pUqLANelfaqBY,3477
-easylink-0.1.17.dist-info/WHEEL,sha256=A8Eltl-h0W-qZDVezsLjjslosEH_pdYC2lQ0JcbgCzs,91
-easylink-0.1.17.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
-easylink-0.1.17.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
-easylink-0.1.17.dist-info/RECORD,,
+easylink-0.1.18.dist-info/METADATA,sha256=9RPc6nIJrkdNQxUXqVYQW26h2G3ukGuXyAmUA4razpA,3477
+easylink-0.1.18.dist-info/WHEEL,sha256=QZxptf4Y1BKFRCEDxD4h2V0mBFQOVFLFEpvxHmIs52A,91
+easylink-0.1.18.dist-info/entry_points.txt,sha256=OGMZDFltg3yMboT7XjJt3joiPhRfV_7jnREVtrAIQNU,51
+easylink-0.1.18.dist-info/top_level.txt,sha256=oHcOpcF_jDMWFiJRzfGQvuskENGDjSPC_Agu9Z_Xvik,9
+easylink-0.1.18.dist-info/RECORD,,

{easylink-0.1.17.dist-info → easylink-0.1.18.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.0)
+Generator: setuptools (80.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any