ceci 2.2__tar.gz → 2.3__tar.gz

{ceci-2.2 → ceci-2.3}/.gitignore

@@ -9,7 +9,6 @@ dist
 log.txt
 test_save*
 tests/log*
-tests/cwl
 nb/Untitled.ipynb
 .ipynb_checkpoints/
 .coverage

{ceci-2.2 → ceci-2.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ceci
-Version: 2.2
+Version: 2.3
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License
@@ -15,14 +15,14 @@ Classifier: Programming Language :: Python :: 3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml>=5.1
+Requires-Dist: ruamel.yaml
 Requires-Dist: psutil
 Requires-Dist: graphlib_backport; python_version < "3.9"
+Requires-Dist: networkx
+Requires-Dist: jinja2
 Provides-Extra: parsl
 Requires-Dist: parsl>=1.0.0; extra == "parsl"
 Requires-Dist: flask; extra == "parsl"
-Provides-Extra: cwl
-Requires-Dist: cwlref-runner; extra == "cwl"
-Requires-Dist: cwl-utils; extra == "cwl"
 Provides-Extra: viz
 Requires-Dist: pygraphviz; extra == "viz"
 Provides-Extra: dask
@@ -38,8 +38,6 @@ Requires-Dist: dask[distributed]; extra == "test"
 Provides-Extra: all
 Requires-Dist: parsl>=1.0.0; extra == "all"
 Requires-Dist: flask; extra == "all"
-Requires-Dist: cwlref-runner; extra == "all"
-Requires-Dist: cwl-utils; extra == "all"
 Requires-Dist: pygraphviz; extra == "all"
 Requires-Dist: pytest; extra == "all"
 Requires-Dist: pytest-cov; extra == "all"
@@ -48,6 +46,8 @@ Requires-Dist: mockmpi; extra == "all"
 Requires-Dist: h5py; extra == "all"
 Requires-Dist: dask[distributed]>=2023.5.0; extra == "all"
 Requires-Dist: dask_mpi>=2022.4.0; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Dynamic: license-file
 
 ![Ceci Logo](ceci.png)
 

ceci-2.3/ceci/__init__.py

@@ -0,0 +1,18 @@
+"""Ceci n'est pas une pipeline"""
+
+from .stage import PipelineStage
+from .pipeline import (
+    Pipeline,
+    MiniPipeline,
+    ParslPipeline,
+    DryRunPipeline,
+    FlowChartPipeline,
+)
+from .main import run_pipeline
+import importlib.metadata
+
+try:
+    __version__ = importlib.metadata.metadata(__name__)["Version"]
+except:  # pragma: no cover
+    # package is not installed
+    pass

ceci-2.3/ceci/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '2.3'
+__version_tuple__ = version_tuple = (2, 3)
+
+__commit_id__ = commit_id = 'ged188fcdb'

{ceci-2.2 → ceci-2.3}/ceci/main.py

@@ -3,12 +3,8 @@
 import os
 import sys
 import argparse
-import subprocess
 from .pipeline import Pipeline
-from .sites import load, set_default_site, get_default_site
-from .utils import extra_paths
 from ._version import __version__
-import contextlib
 
 # Add the current dir to the path - often very useful
 sys.path.append(os.getcwd())
@@ -33,7 +29,15 @@ parser.add_argument(
 parser.add_argument(
     "extra_config",
     nargs="*",
-    help="Over-ride the main pipeline yaml file e.g. launcher.name=cwl",
+    help="Over-ride the main pipeline yaml file e.g. launcher.name=parsl",
+)
+
+parser.add_argument(
+    "-t",
+    "--template-parameters",
+    nargs="*",
+    help="Template parameters to use in the pipeline configuration file, e.g. -t some_directory=xyz field=north",
+    default="",
 )
 
 parser.add_argument(
@@ -59,54 +63,11 @@ def run_pipeline(pipe_config):
     status : `int`
         Usual unix convention of 0 -> success, non-zero is an error code
     """
-    with prepare_for_pipeline(pipe_config):
-        p = Pipeline.create(pipe_config)
-        status = p.run()
+    p = Pipeline.create(pipe_config)
+    status = p.run()
     return status
 
 
-@contextlib.contextmanager
-def prepare_for_pipeline(pipe_config):
-    """
-    Prepare the paths and launcher needed to read and run a pipeline.
-    """
-
-    # Later we will add these paths to sys.path for running here,
-    # but we will also need to pass them to the sites below so that
-    # they can be added within any containers or other launchers
-    # that we use
-    paths = pipe_config.get("python_paths", [])
-    if isinstance(paths, str):  # pragma: no cover
-        paths = paths.split()
-
-    # Get information (maybe the default) on the launcher we may be using
-    launcher_config = pipe_config.setdefault("launcher", {"name": "mini"})
-    site_config = pipe_config.get("site", {"name": "local"})
-
-    # Pass the paths along to the site
-    site_config["python_paths"] = paths
-    load(launcher_config, [site_config])
-
-    # Python modules in which to search for pipeline stages
-    modules = pipe_config.get("modules", "").split()
-
-    # This helps with testing
-    default_site = get_default_site()
-
-    # temporarily add the paths to sys.path,
-    # but remove them at the end
-    with extra_paths(paths):
-
-        # Import modules. We have to do this because the definitions
-        # of the stages can be inside.
-        for module in modules:
-            __import__(module)
-
-        try:
-            yield
-        finally:
-            set_default_site(default_site)
-
 
 def main():  # pragma: no cover
     """Main function called when ceci is invoked on the command line"""
@@ -116,7 +77,8 @@ def main():  # pragma: no cover
     if args.flow_chart:
         args.dry_run = True
     pipe_config = Pipeline.build_config(
-        args.pipeline_config, args.extra_config, args.dry_run, args.flow_chart
+        args.pipeline_config, args.extra_config, args.dry_run, args.flow_chart,
+        args.template_parameters
     )
     status = run_pipeline(pipe_config)
 

{ceci-2.2 → ceci-2.3}/ceci/minirunner.py

@@ -377,7 +377,7 @@ class Runner:
             # status==0 indicates sucess in job, so free resources
             else:
                 run_time = make_run_time_string(default_timer() - start_time)
-                print(f"Job {job.name} has completed successfully in {run_time} seconds !")
+                print(f"Job {job.name} has completed successfully in {run_time}")
                 # Call back with info about the successful job and how it ran
                 self.callback(
                     EVENT_COMPLETED,

ceci-2.3/ceci/pipeline/__init__.py

@@ -0,0 +1,15 @@
+"""Module with core pipeline functionality"""
+
+from .pipeline import (
+    Pipeline,
+    override_config,
+    RESUME_MODE_RESUME,
+    RESUME_MODE_RESTART,
+    RESUME_MODE_REFUSE,
+)
+from .parsl import ParslPipeline
+from .dry_run import DryRunPipeline
+from .flow_chart import FlowChartPipeline
+from .mini import MiniPipeline
+from .file_manager import FileManager
+from .sec import StageExecutionConfig

ceci-2.3/ceci/pipeline/dry_run.py

@@ -0,0 +1,43 @@
+import sys
+from .pipeline import Pipeline
+from ..utils import embolden
+
+
+class DryRunPipeline(Pipeline):
+    """A pipeline subclass which just does a dry-run, showing which commands
+    would be executed.
+
+    See the base class for almost all behaviour.
+
+    No additional attributes.
+    """
+
+    def initiate_run(self, overall_inputs):
+        return []
+
+    def should_skip_stage(self, stage):
+        return False
+
+    def enqueue_job(self, stage, pipeline_files):
+        outputs = stage.find_outputs(self.run_config["output_dir"])
+        sec = self.stage_execution_config[stage.instance_name]
+
+        cmd = sec.generate_full_command(pipeline_files, outputs, self.stages_config)
+
+        # Replace the first instance of the stage name with bold
+        # text, but only if we are printing to screen. This helps the
+        # eye pick out the stage you want to run.
+        if sys.stdout.isatty():
+            cmd = cmd.replace(stage.instance_name, embolden(stage.instance_name), 1)
+
+        self.run_info.append(cmd)
+        return outputs
+
+    def run_jobs(self):
+        for cmd in self.run_info:
+            print(cmd)
+            print("\n")
+        return 0
+
+    def find_all_outputs(self):
+        return {}

ceci-2.3/ceci/pipeline/file_manager.py

@@ -0,0 +1,73 @@
+class FileManager(dict):
+    """Small class to manage files within a particular Pipeline
+
+    This is a dict which is used for tag to path mapping,
+    but has a couple of additional dicts to manage the reverse mapping and the
+    tag to type mapping.
+
+
+    The tag to path mapping is the thing that the Pipeline uses to set the
+    input paths for downstream stages that use the outputs of earlier stages,
+    i.e., everything in the pipeline can refer to a particular file by tag.
+
+    The tag defaults to the input or output tag as define in the stage class attributes.
+    However, in the case that we want multiple stages of the same class in a pipeline we
+    have to alias the tags so that each stage can write to its own location (and so that
+    downstream stages can pick up that location correctly)
+    """
+
+    def __init__(self):
+        """Constructor, makes empty dictionaries"""
+        self._tag_to_type = {}
+        self._path_to_tag = {}
+        dict.__init__(self)
+
+    def __setitem__(self, tag, path):
+        """Override dict.__setitem__() to also insert the reverse mapping"""
+        dict.__setitem__(self, tag, path)
+        self._path_to_tag[path] = tag
+
+    def insert(self, tag, path=None, ftype=None):
+        """Insert a file, including the path and the file type
+
+        Parameters
+        ----------
+        tag : str
+            The tag by which this file will be identified
+        path : str
+            The path to this file
+        ftype : type
+            The file type for this file
+        """
+        if path is not None:
+            self[tag] = path
+            self._path_to_tag[path] = tag
+        if tag not in self._tag_to_type:
+            self._tag_to_type[tag] = ftype
+
+    def get_type(self, tag):
+        """Return the file type associated to a given tag"""
+        return self._tag_to_type[tag]
+
+    def get_path(self, tag):
+        """Return the path associated to a give tag"""
+        return self[tag]
+
+    def get_tag(self, path):
+        """Return the tag associated to a given path"""
+        return self._path_to_tag[path]
+
+    def insert_paths(self, path_dict):
+        """Insert a set of paths from a dict that has tag, path pairs"""
+        for key, val in path_dict.items():
+            self.insert(key, path=val)
+
+    def insert_outputs(self, stage, outdir):
+        """Insert a set of tags and associated paths and file types that are output by a stage"""
+        stage_outputs = stage.find_outputs(outdir)
+        for tag, ftype in stage.outputs:
+            aliased_tag = stage.get_aliased_tag(tag)
+            path = stage_outputs[aliased_tag]
+            self.insert(aliased_tag, path=path, ftype=ftype)
+        return stage_outputs
+

ceci-2.3/ceci/pipeline/flow_chart.py

@@ -0,0 +1,8 @@
+from .dry_run import DryRunPipeline
+
+class FlowChartPipeline(DryRunPipeline):
+    def run_jobs(self):
+        filename = self.run_config["flow_chart"]
+        self.make_flow_chart(filename)
+        return 0
+

ceci-2.3/ceci/pipeline/graph.py

@@ -0,0 +1,199 @@
+import networkx
+import collections
+
+def build_graph(stage_names, stage_classes, stage_aliases, overall_inputs):
+    """
+    Build a directed graph of the pipeline stages and their inputs and outputs.
+
+    This also checks for various pipeline problems like missing inputs,
+    multiple stages generating the same output, and cyclic dependencies.
+    
+    Parameters
+    ----------
+    stage_names : `list` of `str`
+        The names of the stages in the pipeline
+    stage_classes : `list` of `PipelineStage` subclasses
+        The classes of the stages in the pipeline
+    stage_aliases : `list` of `dict`
+        A list of dictionaries mapping tags to aliases for each stage
+    overall_inputs : `dict` of `str: str`
+        A list of overall input files that are not generated by any stage but are needed by the pipeline
+
+    Returns
+    -------
+    graph : `networkx.DiGraph`
+        A directed graph representing the pipeline stages, their inputs, and outputs
+    """
+
+    # Build a graph with all the stages and files in it
+    graph = networkx.DiGraph()
+
+    # First add all the overall inputs
+    for inp in overall_inputs:
+        graph.add_node(inp, type="input")
+    
+    # Now go through all the stages. We do this in two passes,
+    # one to add the stages themselves and their outputs, and one
+    # to connect in all their inputs
+    for stage_name, stage_class, aliases in zip(stage_names, stage_classes, stage_aliases):
+        if stage_name in graph.nodes:
+            raise ValueError(f"Stage {stage_name} is used multiple times in the pipeline")
+        graph.add_node(stage_name, type="stage", stage_class=stage_class)
+
+        # Now set up all the outputs of the stage, first checking to see if it has been aliased
+        for tag1 in stage_class.output_tags():
+            tag = aliases.get(tag1, tag1)
+            if tag in overall_inputs:
+                raise ValueError(f"Tag {tag} is generated by pipeline stage {stage_name} but is also listed as an overall input to the pipeline")
+            graph.add_node(tag, type="output", creator=stage_class, original_tag=tag1)
+            graph.add_edge(stage_name, tag)
+
+    # Now check for outputs that are generated by multiple stages
+    for node in graph.nodes:
+        if graph.nodes[node]["type"] == "output":
+            parents = list(graph.predecessors(node))
+            if len(parents) > 1:
+                raise ValueError(f"Output {node} is generated by multiple stages: {parents}")
+            
+
+    # Now go through and add in all the inputs needed for each stage:
+    missing = collections.defaultdict(list)
+    for stage_name, stage_class, aliases in zip(stage_names, stage_classes, stage_aliases):
+        for tag in stage_class.input_tags():
+            tag = aliases.get(tag, tag)
+            if tag not in graph.nodes:
+                missing[tag].append(stage_name)
+            else:
+                graph.add_edge(tag, stage_name)
+    
+    if missing:
+        msg = []
+        for tag, stages in missing.items():
+            stages = ", ".join(stages)
+            msg.append(f"- {tag} - needed by stages {stages}")
+        msg = "\n.join(msg)"
+        raise ValueError(f"The following files are needed by the pipeline but not generated by any stage:\n{msg}")
+
+    # Now check for cycles in the graph
+    cycles = list(networkx.simple_cycles(graph))
+    if cycles:
+        msg = []
+        for cycle in cycles:
+            cycle_text = " -> ".join(cycle)
+            cycle_text += " -> " + cycle[0]
+            msg.append(f"- {cycle_text}")
+        raise ValueError(f"Your pipeline is cyclic. The following cycles were found:\n{msg}")
+    
+    return graph
+
+def get_static_ordering(graph):
+    # get a topological ordering of the graph
+    # this will be a list of all the nodes in the graph, including the input and output files
+    ordering = list(networkx.topological_sort(graph))
+
+    # cut down to just the stages
+    ordering = [node for node in ordering if graph.nodes[node]["type"] == "stage"]
+   
+    return ordering
+
+def trim_pipeline_graph(graph, from_=None, to_=None):
+    """
+    Trim the pipeline graph to only include the stages between `from_` and `to_`.
+
+    Either `from_` or `to_` can be `None`, in which case no trimming is done in that direction.
+
+    They can be single strings, or lists of strings, in which case the graph is trimmed to include all stages
+    between any of the stages in `from_` and any of the stages in `to
+
+    Parameters
+    ----------
+    graph : `networkx.DiGraph`
+        The original pipeline graph
+    from_ : `str`
+        The name of the stage(s) or file(s) to start from.
+    to_ : `str`
+        The name of the stage(s) or file(s) to to end at.
+
+    Returns
+    -------
+    trimmed_graph : `networkx.DiGraph`
+        The trimmed pipeline graph
+    """
+    if isinstance(from_, str):
+        from_ = [from_]
+    if isinstance(to_, str):
+        to_ = [to_]
+
+    # Descendents is all nodes that are descendend from any of the nodes in `from_`
+    # if from_ is None then it includes all nodes in the graph
+    descendents = set()
+
+    if from_ is None:
+        descendents.update(graph.nodes)
+    else:
+        for node in from_:
+            if node not in graph.nodes:
+                raise ValueError(f"Node {node} is not in the pipeline graph")
+            # first include the nodes in `from_` themselves
+            descendents.add(node)
+            descendents.update(networkx.descendants(graph, node))
+
+    # Ancestors is all nodes that are ancestors of any of the nodes in `to_`
+    # if to_ is None then it includes all nodes in the graph
+    ancestors = set()
+    if to_ is None:
+        ancestors.update(graph.nodes)
+    else:
+        for node in to_:
+            if node not in graph.nodes:
+                raise ValueError(f"Node {node} is not in the pipeline graph")
+            ancestors.add(node)
+            ancestors.update(networkx.ancestors(graph, node))
+
+    # Now we want to keep everything that is both a descendent of `from_` and an ancestor of `to_`
+    # This is the intersection of the two sets
+    nodes_to_keep = descendents.intersection(ancestors)
+
+    # We also want to keep the input and output files for the stages we are keeping
+    # (though not the stages that make those)
+    more_to_keep = set()
+    for node in nodes_to_keep:
+        if graph.nodes[node]["type"] == "stage":
+            # Get all the outputs of this stage
+            outputs = [n for n in graph.successors(node) if graph.nodes[n]["type"] == "output"]
+            more_to_keep.update(outputs)
+            # same for the inputs to all the stages
+            inputs = [n for n in graph.predecessors(node) if graph.nodes[n]["type"] in ["input", "output"]]
+            more_to_keep.update(inputs)
+    nodes_to_keep.update(more_to_keep)
+
+    # Create a subgraph with just these nodes
+    trimmed_graph = graph.subgraph(nodes_to_keep).copy()
+
+    # Some stages that were previously outputs of existing stages
+    # must now be overall inputs to the pipeline instead because
+    # the stage that generated them has been trimmed. We need to keep
+    # track of this
+    converted_inputs = {}
+    for node_name in list(trimmed_graph.nodes):
+        node = trimmed_graph.nodes[node_name]
+
+        if node["type"] == "output":
+            # The node will be listed as an output but will have no
+            # predecessor creating it now.
+            predecessors = list(trimmed_graph.predecessors(node_name))
+            if not predecessors:
+                creator_stage_class = node["creator"]
+                original_tag = node["original_tag"]
+                # use the original tag to find the file type object
+                for t, typ in creator_stage_class.outputs:
+                    if t == original_tag:
+                        ftype = typ
+                        break
+                else:
+                    raise RuntimeError("This should never happen - please open an issue")
+                # the node_name is the aliased_tag, so this file name should be correct
+                file_name = ftype.make_name(node_name)
+                converted_inputs[node_name] = file_name
+
+    return trimmed_graph, converted_inputs

ceci-2.3/ceci/pipeline/mini.py

@@ -0,0 +1,133 @@
+import sys
+from .. import minirunner
+from .pipeline import Pipeline, get_default_site
+
+class MiniPipeline(Pipeline):
+    """A pipeline subclass that uses Minirunner, a sub-module
+    of ceci, to run.
+
+    Minirununer is a small tool I wrote suitable for interactive
+    jobs on cori.  It launches jobs locally (not through a batch system),
+    which parsl can also do, but it has a simple and clearer (to me at least)
+    understanding of available nodes and cores.
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        """Create a MiniRunner Pipeline
+
+        In addition to parent initialization parameters (see the
+        Pipeline base class), this subclass can take these optional
+        keywords.
+
+        Parameters
+        ----------
+        callback: function(event_type: str, event_info: dict)
+            A function called when jobs launch, complete, or fail,
+            and when the pipeline aborts.  Can be used for tracing
+            execution.  Default=None.
+
+        sleep: function(t: float)
+            A function to replace time.sleep called in the pipeline
+            to wait until the next time to check process completion
+            Most normal usage will not need this.  Default=None.
+        """
+        self.callback = kwargs.pop("callback", None)
+        self.sleep = kwargs.pop("sleep", None)
+        super().__init__(*args, **kwargs)
+
+    def build_dag(self, jobs):
+        """Build a directed acyclic graph of a set of stages.
+
+        The DAG is represented by a list of jobs that each other job
+        depends on.  If all a job's dependencies are complete
+        then it can be run.
+
+        Fun fact: the word "dag" is also Australian slang for a
+        "dung-caked lock of wool around the hindquarters of a sheep".
+        and is used as a semi-affectionate insult.
+
+        Parameters
+        ----------
+
+        stages: list[PipelineStage]
+            A list of stages to generate the DAG for
+
+        """
+        depend = {}
+
+        # for each stage in our pipeline ...
+        for stage in self.stages[:]:
+            if stage.instance_name not in jobs:
+                continue
+            job = jobs[stage.instance_name]
+            depend[job] = []
+            # check for each of the inputs for that stage ...
+            for tag in stage.input_tags():
+                aliased_tag = stage.get_aliased_tag(tag)
+                for potential_parent in self.stages[:]:
+                    # if that stage is supplied by another pipeline stage
+                    if potential_parent.instance_name not in jobs:  # pragma: no cover
+                        continue
+                    potential_parent_tags = [
+                        potential_parent.get_aliased_tag(tag_)
+                        for tag_ in potential_parent.output_tags()
+                    ]
+                    if aliased_tag in potential_parent_tags:
+                        depend[job].append(jobs[potential_parent.instance_name])
+        return depend
+
+    def initiate_run(self, overall_inputs):
+        jobs = {}
+        stages = []
+        return jobs, stages
+
+    def enqueue_job(self, stage, pipeline_files):
+        sec = self.stage_execution_config[stage.instance_name]
+        outputs = stage.find_outputs(self.run_config["output_dir"])
+        cmd = sec.generate_full_command(pipeline_files, outputs, self.stages_config)
+        job = minirunner.Job(
+            stage.instance_name,
+            cmd,
+            cores=sec.threads_per_process * sec.nprocess,
+            nodes=sec.nodes,
+        )
+        self.run_info[0][stage.instance_name] = job
+        self.run_info[1].append(stage)
+        return outputs
+
+    def run_jobs(self):
+        jobs, _ = self.run_info
+        # Now the jobs have all been queued, build them into a graph
+        graph = self.build_dag(jobs)
+        nodes = get_default_site().info["nodes"]
+        log_dir = self.run_config["log_dir"]
+
+        # This pipeline, being mainly for testing, can only
+        # run at a single site, so we can assume here that the
+        # sites are all the same
+        sec = self.stage_execution_config[self.stage_names[0]]
+        nodes = sec.site.info["nodes"]
+
+        # Run under minirununer
+        runner = minirunner.Runner(
+            nodes, graph, log_dir, callback=self.callback, sleep=self.sleep
+        )
+        interval = self.launcher_config.get("interval", 3)
+        try:
+            runner.run(interval)
+        except minirunner.FailedJob as error:
+            sys.stderr.write(
+                f"""
+*************************************************
+Error running pipeline stage {error.job_name}.
+Failed after {error.run_time}.
+
+Standard output and error streams in {log_dir}/{error.job_name}.out
+*************************************************
+"""
+            )
+            return 1
+
+        return 0
+