easylink 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

easylink/_version.py

@@ -1 +1 @@
-__version__ = "0.1.17"
+__version__ = "0.1.19"

easylink/cli.py

@@ -55,7 +55,7 @@ from easylink.utilities.general_utils import (
     configure_logging_to_terminal,
     handle_exceptions,
 )
-from easylink.utilities.paths import CONTAINER_DIR
+from easylink.utilities.paths import DEFAULT_IMAGES_DIR, DEV_IMAGES_DIR
 
 SHARED_OPTIONS = [
     click.option(
@@ -91,6 +91,11 @@ SHARED_OPTIONS = [
         default=False,
         help="Do not save the results in a timestamped sub-directory of ``--output-dir``.",
     ),
+    click.option(
+        "--schema",
+        hidden=True,
+        default="main",
+    ),
 ]
 
 VERBOSE_WITH_DEBUGGER_OPTIONS = [
@@ -149,6 +154,16 @@ def easylink():
 
 @easylink.command()
 @_pass_shared_options
+@click.option(
+    "-I",
+    "--images",
+    hidden=True,
+    type=click.Path(exists=False, file_okay=False, resolve_path=True),
+    help=(
+        "The directory containing the images to run. If no value is passed, a new "
+        f"directory will be created at the home directory: {DEFAULT_IMAGES_DIR}."
+    ),
+)
 @click.option(
     "-e",
     "--computing-environment",
@@ -165,6 +180,8 @@ def run(
     input_data: str,
     output_dir: str | None,
     no_timestamp: bool,
+    schema: str,
+    images: str,
     computing_environment: str | None,
     verbose: int,
     with_debugger: bool,
@@ -190,6 +207,8 @@ def run(
         input_data=input_data,
         computing_environment=computing_environment,
         results_dir=results_dir,
+        images_dir=images,
+        schema_name=schema,
     )
     logger.info("*** FINISHED ***")
 
@@ -201,6 +220,7 @@ def generate_dag(
     input_data: str,
     output_dir: str | None,
     no_timestamp: bool,
+    schema: str,
     verbose: int,
     with_debugger: bool,
 ) -> None:
@@ -223,6 +243,7 @@ def generate_dag(
         input_data=input_data,
         computing_environment=None,
         results_dir=results_dir,
+        schema_name=schema,
     )
     logger.info("*** DAG saved to result directory ***")
 
@@ -254,7 +275,7 @@ easylink.add_command(devtools)
     type=click.Path(exists=False, dir_okay=True, file_okay=False, resolve_path=True),
     help=(
         "The directory to move the container to. If no value is passed, it will "
-        f"be moved to {CONTAINER_DIR} in a sub-directory named with the username."
+        f"be moved to {DEV_IMAGES_DIR} in a sub-directory named with the username."
     ),
 )
 def create_implementation(
@@ -291,7 +312,7 @@ def create_implementation(
     if not scripts:
         logger.error("No scripts provided.")
         return
-    output_dir = Path(output_dir) if output_dir else Path(f"{CONTAINER_DIR}/{os.getlogin()}")
+    output_dir = Path(output_dir) if output_dir else Path(f"{DEV_IMAGES_DIR}/{os.getlogin()}")
     if not output_dir.exists():
         # make the directory with rwxrwxr-x permissions
         output_dir.mkdir(parents=True, mode=0o775)

easylink/configuration.py

@@ -14,9 +14,10 @@ from typing import Any
 
 from layered_config_tree import LayeredConfigTree
 
-from easylink.pipeline_schema import PIPELINE_SCHEMAS, PipelineSchema
+from easylink.pipeline_schema import PipelineSchema
 from easylink.utilities.data_utils import load_yaml
 from easylink.utilities.general_utils import exit_with_validation_error
+from easylink.utilities.paths import DEFAULT_IMAGES_DIR
 
 PIPELINE_ERRORS_KEY = "PIPELINE ERRORS"
 INPUT_DATA_ERRORS_KEY = "INPUT DATA ERRORS"
@@ -66,10 +67,14 @@ class Config(LayeredConfigTree):
     config_params
         A dictionary of all specifications required to run the pipeline. This
         includes the pipeline, input data, and computing environment specifications,
-        as well as the results directory.
-    potential_schemas
-        A list of potential schemas to validate the pipeline configuration against.
-        This is primarily used for testing purposes. Defaults to the supported schemas.
+        as well as the results directory and images directory.
+    schema_name
+        The name of the schema to validate the pipeline configuration against.
+    images_dir
+        The directory containing the images or to download the images to if they
+        don't exist. If None, will default to the :data:`~easylink.utilities.paths.DEFAULT_IMAGES_DIR`.
+    command
+        The EasyLink command being run.
 
     Attributes
     ----------
@@ -82,22 +87,21 @@ class Config(LayeredConfigTree):
     input_data
         The input data filepaths.
     schema
-        The :class:`~easylink.pipeline_schema.PipelineSchema` that successfully
-        validated the requested pipeline.
-
-    Notes
-    -----
-    The requested pipeline is checked against a set of supported
-    ``PipelineSchemas``. The first schema that successfully validates is assumed
-    to be the correct one and is attached to the ``Config`` object and its
-    :meth:`~easylink.pipeline_schema.PipelineSchema.configure_pipeline`
-    method is called.
+        The :class:`~easylink.pipeline_schema.PipelineSchema`.
+    images_dir
+        The directory containing the images or to download the images to if they
+        don't exist. If None, will default to ~/.easylink_images.
+    command
+        The EasyLink command being run.
+
     """
 
     def __init__(
         self,
         config_params: dict[str, Any],
-        potential_schemas: PipelineSchema | list[PipelineSchema] = PIPELINE_SCHEMAS,
+        schema_name: str = "main",
+        images_dir: str | Path | None = None,
+        command: str = "run",
     ) -> None:
         super().__init__(layers=["initial_data", "default", "user_configured"])
         self.update(DEFAULT_ENVIRONMENT, layer="default")
@@ -108,10 +112,16 @@ class Config(LayeredConfigTree):
             # Set slurm defaults to empty dict instead of None so that we don't get errors
             # in slurm_resources property
             self.update({"environment": {"slurm": {}}}, layer="default")
-        if not isinstance(potential_schemas, list):
-            potential_schemas = [potential_schemas]
-        self.update({"schema": self._get_schema(potential_schemas)}, layer="initial_data")
+        self.update({"schema": self._get_schema(schema_name)}, layer="initial_data")
         self.schema.configure_pipeline(self.pipeline, self.input_data)
+        # use the images_dir if provided, otherwise use default
+        self.update(
+            {
+                "images_dir": Path(images_dir) if images_dir else DEFAULT_IMAGES_DIR,
+            },
+            layer="user_configured",
+        )
+        self.update({"command": command}, layer="user_configured")
         self._validate()
         self.freeze()
 
@@ -173,22 +183,22 @@ class Config(LayeredConfigTree):
     # Setup Methods #
     #################
 
-    def _get_schema(self, potential_schemas: list[PipelineSchema]) -> PipelineSchema:
+    def _get_schema(self, schema_name: str = "main") -> PipelineSchema:
         """Returns the first :class:`~easylink.pipeline_schema.PipelineSchema` that validates the requested pipeline.
 
         Parameters
         ----------
-        potential_schemas
-            ``PipelineSchemas`` to validate the pipeline configuration against.
+        schema_name
+            The name of the specific ``PipelineSchema`` to validate the pipeline configuration against.
 
         Returns
         -------
-            The first ``PipelineSchema`` that validates the requested pipeline configuration.
+            The requested ``PipelineSchema`` if it validates the requested pipeline configuration.
 
         Raises
         ------
         SystemExit
-            If the pipeline configuration is not valid for any of the ``potential_schemas``,
+            If the pipeline configuration is not valid for the requested schema,
             the program exits with a non-zero code and all validation errors found
             are logged.
 
@@ -197,20 +207,15 @@ class Config(LayeredConfigTree):
         This acts as the pipeline configuration file's validation method since
         we can only find a matching ``PipelineSchema`` if that file is valid.
 
-        This method returns the *first* ``PipelineSchema`` that validates and does
-        not attempt to check additional ones.
         """
         errors = defaultdict(dict)
         # Try each schema until one is validated
-        for schema in potential_schemas:
-            logs = schema.validate_step(self.pipeline, self.input_data)
-            if logs:
-                errors[PIPELINE_ERRORS_KEY][schema.name] = logs
-                pass  # try the next schema
-            else:  # schema was validated
-                return schema
-        # No schemas were validated
-        exit_with_validation_error(dict(errors))
+        schema = PipelineSchema.get_schema(schema_name)
+        logs = schema.validate_step(self.pipeline, self.input_data)
+        if logs:
+            errors[PIPELINE_ERRORS_KEY][schema.name] = logs
+            exit_with_validation_error(dict(errors))
+        return schema
 
     def _validate(self) -> None:
         """Validates the ``Config``.
@@ -319,7 +324,9 @@ def _load_input_data_paths(
             f"Input was: '{input_data_paths}'"
         )
     filepath_dict = {
-        filename: Path(filepath).resolve() for filename, filepath in input_data_paths.items()
+        # Resolve paths relative to location of the YAML file
+        filename: (Path(input_data_specification_path).parent / Path(filepath)).resolve()
+        for filename, filepath in input_data_paths.items()
     }
     return filepath_dict
 

easylink/devtools/implementation_creator.py

@@ -19,7 +19,7 @@ from typing import cast
 import yaml
 from loguru import logger
 
-from easylink.pipeline_schema_constants import ALLOWED_SCHEMA_PARAMS
+from easylink.pipeline_schema_constants import SCHEMA_PARAMS
 from easylink.step import (
     ChoiceStep,
     EmbarrassinglyParallelStep,
@@ -69,8 +69,6 @@ class ImplementationCreator:
         for the container.
     implementation_name
         The name of the implementation. It is by definition the name of the script.
-    requirements
-        The install requirements for the implementation (if any).
     step
         The name of the step that this implementation implements.
     output_slot
@@ -93,20 +91,30 @@ class ImplementationCreator:
         for the container."""
         self.implementation_name = script_path.stem
         """The name of the implementation. It is by definition the name of the script."""
-        self.requirements = self._extract_requirements(script_path)
-        """The install requirements for the implementation (if any)."""
         self.step = self._extract_implemented_step(script_path)
         """The name of the step that this implementation implements."""
+        self.has_custom_recipe = self._extract_has_custom_recipe(script_path)
+        """Whether the user has already written the recipe for this implementation."""
+        self.script_base_command = self._extract_script_base_command(script_path)
+        """The base command to use to run the script in this implementation."""
         self.output_slot = self._extract_output_slot(script_path, self.step)
         """The name of the output slot that this implementation sends results to."""
 
     def create_recipe(self) -> None:
         """Builds the singularity recipe and writes it to disk."""
-
-        recipe = PythonRecipe(self.script_path, self.recipe_path, self.requirements)
+        if self.has_custom_recipe:
+            if not self.recipe_path.exists():
+                raise ValueError(f"Could not find a custom recipe at {self.recipe_path}.")
+            return
+
+        recipe = PythonRecipe(
+            self.script_path,
+            self.recipe_path,
+            ImplementationCreator._extract_requirements(self.script_path),
+            self.script_base_command,
+        )
         recipe.build()
         recipe.write()
-        pass
 
     def build_container(self) -> None:
         """Builds the container from the recipe.
@@ -190,7 +198,7 @@ class ImplementationCreator:
         info[self.implementation_name] = {
             "steps": [self.step],
             "image_path": str(self.hosted_container_path),
-            "script_cmd": f"python /{self.script_path.name}",
+            "script_cmd": f"{self.script_base_command} /{self.script_path.name}",
             "outputs": {
                 self.output_slot: "result.parquet",
             },
@@ -241,20 +249,36 @@ class ImplementationCreator:
             )
         return steps[0]
 
+    @staticmethod
+    def _extract_has_custom_recipe(script_path: Path) -> bool:
+        """Extracts whether the user has already written the recipe for this implementation.
+
+        The expectation is that this flag is specified within the script
+        as a comment of the format:
+
+        .. code-block:: python
+            # HAS_CUSTOM_RECIPE: true
+        """
+        has_custom_recipe = _extract_metadata("HAS_CUSTOM_RECIPE", script_path)
+        if len(has_custom_recipe) == 0:
+            return False
+        else:
+            return str(has_custom_recipe[0]).strip().lower() in ["true", "yes"]
+
     @staticmethod
     def _extract_output_slot(script_path: Path, step_name: str) -> str:
         """Extracts the name of the output slot that this script is implementing."""
-        schema = ImplementationCreator._extract_pipeline_schema(script_path)
-        implementable_steps = ImplementationCreator._extract_implementable_steps(schema)
+        schema_name = ImplementationCreator._extract_pipeline_schema_name(script_path)
+        implementable_steps = ImplementationCreator._extract_implementable_steps(schema_name)
         step_names = [step.name for step in implementable_steps]
         if step_name not in step_names:
             raise ValueError(
-                f"'{step_name}' does not exist as an implementable step in the '{schema}' pipeline schema. "
+                f"'{step_name}' does not exist as an implementable step in the '{schema_name}' pipeline schema. "
             )
         duplicates = list(set([step for step in step_names if step_names.count(step) > 1]))
         if duplicates:
             raise ValueError(
-                f"Multiple implementable steps with the same name found in the '{schema}' "
+                f"Multiple implementable steps with the same name found in the '{schema_name}' "
                 f"pipeline schema: {duplicates}."
             )
         implemented_step = [step for step in implementable_steps if step.name == step_name][0]
@@ -266,7 +290,7 @@ class ImplementationCreator:
         return list(implemented_step.output_slots)[0]
 
     @staticmethod
-    def _extract_implementable_steps(schema: str) -> list[Step]:
+    def _extract_implementable_steps(schema_name: str) -> list[Step]:
         """Extracts all implementable steps from the pipeline schema.
 
         This method recursively traverses the pipeline schema specified in the script
@@ -296,8 +320,7 @@ class ImplementationCreator:
                 implementable_steps.append(node)
                 return
 
-        schema_steps = ALLOWED_SCHEMA_PARAMS[schema][0]
-
+        schema_steps, _edges = SCHEMA_PARAMS[schema_name]
         implementable_steps: list[Step] = []
         for schema_step in schema_steps:
             _process_step(schema_step)
@@ -305,10 +328,10 @@ class ImplementationCreator:
         return implementable_steps
 
     @staticmethod
-    def _extract_pipeline_schema(script_path: Path) -> str:
+    def _extract_pipeline_schema_name(script_path: Path) -> str:
         """Extracts the relevant pipeline schema name.
 
-        The expectation is that the output slot's name is specified within the script
+        The expectation is that the pipeline schema's name is specified within the script
         as a comment of the format:
 
         .. code-block:: python
@@ -316,8 +339,27 @@ class ImplementationCreator:
 
         If no pipeline schema is specified, "main" will be used by default.
         """
-        schema = _extract_metadata("PIPELINE_SCHEMA", script_path)
-        return "main" if len(schema) == 0 else schema[0]
+        schema_name_list: list[str] = _extract_metadata("PIPELINE_SCHEMA", script_path)
+        schema_name = "main" if len(schema_name_list) == 0 else schema_name_list[0]
+        if schema_name not in SCHEMA_PARAMS:
+            raise ValueError(f"Pipeline schema '{schema_name}' is not supported.")
+        return schema_name
+
+    @staticmethod
+    def _extract_script_base_command(script_path: Path) -> str:
+        """Extracts the base command to be used to run the script.
+
+        The expectation is that the base command is specified within the script
+        as a comment of the format:
+
+        .. code-block:: python
+            # SCRIPT_BASE_COMMAND: python
+
+        If no pipeline schema is specified, "python" will be used by default.
+        """
+        base_command_list: list[str] = _extract_metadata("SCRIPT_BASE_COMMAND", script_path)
+        base_command = base_command_list[0] if base_command_list else "python"
+        return base_command
 
     @staticmethod
     def _write_metadata(info: dict[str, dict[str, str]]) -> None:
@@ -339,10 +381,17 @@ class PythonRecipe:
         "python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899"
     )
 
-    def __init__(self, script_path: Path, recipe_path: Path, requirements: str) -> None:
+    def __init__(
+        self,
+        script_path: Path,
+        recipe_path: Path,
+        requirements: str,
+        script_base_command: str,
+    ) -> None:
         self.script_path = script_path
         self.recipe_path = recipe_path
         self.requirements = requirements
+        self.script_base_command = script_base_command
         self.text: str | None = None
 
     def build(self) -> None:
@@ -371,7 +420,7 @@ From: {self.BASE_IMAGE}
     export LC_ALL=C
 
 %runscript
-    python /{script_name} '$@'"""
+    {self.script_base_command} /{script_name} '$@'"""
 
     def write(self) -> None:
         """Writes the recipe to disk.

easylink/implementation.py

@@ -16,9 +16,14 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 from layered_config_tree import LayeredConfigTree
+from loguru import logger
 
 from easylink.utilities import paths
-from easylink.utilities.data_utils import load_yaml
+from easylink.utilities.data_utils import (
+    calculate_md5_checksum,
+    download_image,
+    load_yaml,
+)
 
 if TYPE_CHECKING:
     from easylink.graph_components import InputSlot, OutputSlot
@@ -74,14 +79,14 @@ class Implementation:
     def __repr__(self) -> str:
         return f"Implementation.{self.name}"
 
-    def validate(self) -> list[str]:
+    def validate(self, skip_image_validation: bool, images_dir: str | Path) -> list[str]:
         """Validates individual ``Implementation`` instances.
 
         Returns
         -------
             A list of logs containing any validation errors. Each item in the list
             is a distinct message about a particular validation error (e.g. if a
-            required container does not exist).
+            required image does not exist).
 
         Notes
         -----
@@ -89,7 +94,8 @@ class Implementation:
         """
         logs = []
         logs = self._validate_expected_steps(logs)
-        logs = self._validate_container_exists(logs)
+        if not skip_image_validation:
+            logs = self._download_and_validate_image(logs, images_dir)
         return logs
 
     ##################
@@ -110,11 +116,82 @@ class Implementation:
             )
         return logs
 
-    def _validate_container_exists(self, logs: list[str]) -> list[str]:
-        """Validates that the container to run exists."""
-        err_str = f"Container '{self.singularity_image_path}' does not exist."
-        if not Path(self.singularity_image_path).exists():
-            logs.append(err_str)
+    def _download_and_validate_image(
+        self, logs: list[str], images_dir: str | Path
+    ) -> list[str]:
+        """Downloads the image if required and validates it exists.
+
+        If the image does not exist in the specified images directory, it will
+        attempt to download it.
+        """
+        # HACK: We manually create the image path here as well as later when writing
+        # each implementations Snakefile rule.
+        image_path = Path(images_dir) / self.singularity_image_name
+        expected_md5_checksum = self._metadata.get("md5_checksum", None)
+        record_id = self._metadata.get("zenodo_record_id", None)
+        if image_path.exists():
+            self._handle_conflicting_checksums(
+                logs, image_path, expected_md5_checksum, record_id
+            )
+        else:
+            if not record_id:
+                logs.append(
+                    f"Image '{str(image_path)}' does not exist and no Zenodo record ID "
+                    "is provided to download it."
+                )
+            if not expected_md5_checksum:
+                logs.append(
+                    f"Image '{str(image_path)}' does not exist and no MD5 checksum "
+                    "is provided to verify from the host."
+                )
+            if not record_id or not expected_md5_checksum:
+                return logs
+            download_image(
+                images_dir=images_dir,
+                record_id=record_id,
+                filename=self.singularity_image_name,
+                md5_checksum=expected_md5_checksum,
+            )
+        if not image_path.exists():
+            logs.append(
+                f"Image '{str(image_path)}' does not exist and could not be downloaded."
+            )
+        return logs
+
+    @staticmethod
+    def _handle_conflicting_checksums(
+        logs: list[str],
+        image_path: Path,
+        expected_md5_checksum: str | None,
+        record_id: str | None,
+    ) -> list[str]:
+        # TODO: Strengthen the following logic to better handle image updates.
+        # If using the default images directory and the image already exists
+        # but with a different checksum than in the implementation metadata,
+        # re-download.
+        calculated_md5_checksum = calculate_md5_checksum(image_path)
+        if (
+            image_path.parent == paths.DEFAULT_IMAGES_DIR
+            and expected_md5_checksum
+            and calculated_md5_checksum != expected_md5_checksum
+        ):
+            if not record_id:
+                logs.append(
+                    f"Image '{str(image_path)}' exists but has a different MD5 checksum "
+                    f"({calculated_md5_checksum}) than expected ({expected_md5_checksum}). "
+                    "No Zenodo record ID is provided to re-download the image."
+                )
+            logger.info(
+                f"Image '{str(image_path)}' exists but has a different MD5 checksum "
+                f"({calculated_md5_checksum}) than expected ({expected_md5_checksum}). "
+                "Re-downloading the image."
+            )
+            download_image(
+                images_dir=image_path.parent,
+                record_id=record_id,
+                filename=image_path.name,
+                md5_checksum=expected_md5_checksum,
+            )
         return logs
 
     def _get_env_vars(self, implementation_config: LayeredConfigTree) -> dict[str, str]:
@@ -124,9 +201,9 @@ class Implementation:
         return env_vars
 
     @property
-    def singularity_image_path(self) -> str:
+    def singularity_image_name(self) -> str:
         """The path to the required Singularity image."""
-        return self._metadata["image_path"]
+        return self._metadata["image_name"]
 
     @property
     def script_cmd(self) -> str: