ceci 2.6__tar.gz → 2.7__tar.gz

{ceci-2.6 → ceci-2.7}/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
   
     steps:
     - name: Checkout repository
@@ -72,7 +72,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.12]
 
     steps:
     - name: Checkout repository
@@ -103,7 +103,7 @@ jobs:
         echo "sqlalchemy.url : postgresql://postgres:postgres@localhost:5432/desc_data_registry" > $DATAREG_CONFIG
         # Install the data registry
         cd dataregistry
-        python3 -m pip install .
+        python3 -m pip install . psycopg
         # Create the data registry schema
         python3 scripts/create_registry_db.py --config $DATAREG_CONFIG
         # Create some dummy data registry entries

{ceci-2.6 → ceci-2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.6
+Version: 2.7
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.6 → ceci-2.7}/ceci/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.6'
-__version_tuple__ = version_tuple = (2, 6)
+__version__ = version = '2.7'
+__version_tuple__ = version_tuple = (2, 7)
 
-__commit_id__ = commit_id = 'gcf592d97c'
+__commit_id__ = commit_id = 'g6fabd7fa4'

{ceci-2.6 → ceci-2.7}/ceci/pipeline/__init__.py

@@ -11,5 +11,5 @@ from .parsl import ParslPipeline
 from .dry_run import DryRunPipeline
 from .flow_chart import FlowChartPipeline
 from .mini import MiniPipeline
-from .file_manager import FileManager
 from .sec import StageExecutionConfig
+from .graph import StageStatus

{ceci-2.6 → ceci-2.7}/ceci/pipeline/dry_run.py

@@ -13,16 +13,19 @@ class DryRunPipeline(Pipeline):
     """
 
     def initiate_run(self, overall_inputs):
-        return []
+        return {
+            "commands": [],
+            "completed_stages": [],
+        }
 
     def should_skip_stage(self, stage):
         return False
 
-    def enqueue_job(self, stage, pipeline_files):
+    def enqueue_job(self, stage, input_paths):
         outputs = stage.find_outputs(self.run_config["output_dir"])
-        sec = self.stage_execution_config[stage.instance_name]
+        sec = self.get_stage_execution_config(stage.instance_name)
 
-        cmd = sec.generate_full_command(pipeline_files, outputs, self.stages_config)
+        cmd = sec.generate_full_command(input_paths, outputs, self.stages_config)
 
         # Replace the first instance of the stage name with bold
         # text, but only if we are printing to screen. This helps the
@@ -30,13 +33,20 @@ class DryRunPipeline(Pipeline):
         if sys.stdout.isatty():
             cmd = cmd.replace(stage.instance_name, embolden(stage.instance_name), 1)
 
-        self.run_info.append(cmd)
+        # make a list of complete stages and print at the end
+        if self.run_config["resume"] in (True, "resume") and super().should_skip_stage(stage):
+            self.run_info["completed_stages"].append(stage.instance_name)
+
+        self.run_info["commands"].append(cmd)
         return outputs
 
     def run_jobs(self):
-        for cmd in self.run_info:
+        for cmd in self.run_info["commands"]:
             print(cmd)
             print("\n")
+        print("\n")
+        for name in self.run_info["completed_stages"]:
+            print(f"# {name} is complete and could be skipped")
         return 0
 
     def find_all_outputs(self):

ceci-2.7/ceci/pipeline/flow_chart.py

@@ -0,0 +1,16 @@
+from .dry_run import DryRunPipeline
+
+class FlowChartPipeline(DryRunPipeline):
+    def run_jobs(self):
+        filename = self.run_config["flow_chart"]
+        self.make_flow_chart(filename)
+        return 0
+
+    def setup_data_registry(self, registry_config):
+        """Flow charts never read data, so do not connect to the data registry."""
+        return None
+
+    def data_registry_lookup(self, info):
+        """Return a placeholder path instead of querying the data registry."""
+        return f"registry:{info}"
+

{ceci-2.6 → ceci-2.7}/ceci/pipeline/graph.py

@@ -1,71 +1,156 @@
+import enum
 import networkx
 import collections
 
-def build_graph(stage_names, stage_classes, stage_aliases, overall_inputs):
+
+class StageStatus(enum.Enum):
+    """The run status of a pipeline stage, stored on its graph node"""
+
+    PENDING = "pending"
+    QUEUED = "queued"
+    RUNNING = "running"
+    SKIPPED = "skipped"
+    COMPLETE = "complete"
+    FAILED = "failed"
+
+
+# Key in the graph attributes dictionary where the static stage ordering
+# is cached. Functions in this module that change the stages or connections
+# in a graph invalidate it.
+STATIC_ORDERING = "static_ordering"
+
+
+def invalidate_static_ordering(graph):
+    """Clear the cached static ordering of a graph, if there is one.
+
+    This must be called when stages or edges are added to or removed from
+    the graph other than through the functions in this module.
+
+    Parameters
+    ----------
+    graph : `networkx.DiGraph`
+        The graph to modify
     """
-    Build a directed graph of the pipeline stages and their inputs and outputs.
+    graph.graph.pop(STATIC_ORDERING, None)
+
+
+def add_stage_node(graph, stage_name, stage_class, aliases, **attrs):
+    """
+    Add a stage, and the output files it generates, to a pipeline graph.
+
+    The inputs to the stage are not connected here, since they may not
+    exist in the graph yet; use `connect_stage_inputs` for that.
 
-    This also checks for various pipeline problems like missing inputs,
-    multiple stages generating the same output, and cyclic dependencies.
-    
     Parameters
     ----------
-    stage_names : `list` of `str`
-        The names of the stages in the pipeline
-    stage_classes : `list` of `PipelineStage` subclasses
-        The classes of the stages in the pipeline
-    stage_aliases : `list` of `dict`
-        A list of dictionaries mapping tags to aliases for each stage
-    overall_inputs : `dict` of `str: str`
-        A list of overall input files that are not generated by any stage but are needed by the pipeline
+    graph : `networkx.DiGraph`
+        The graph to add to
+    stage_name : `str`
+        The name of the stage
+    stage_class : `PipelineStage` subclass
+        The class of the stage
+    aliases : `dict`
+        Dictionary mapping tags to aliases for this stage
+    **attrs
+        Any additional attributes to set on the stage node
+    """
+    if stage_name in graph.nodes:
+        raise ValueError(f"Stage {stage_name} is used multiple times in the pipeline")
+
+    # Check all the outputs before changing anything, so that
+    # a failure leaves the graph unchanged
+    outputs = []
+    for tag1, ftype in stage_class.outputs_():
+        tag = aliases.get(tag1, tag1)
+        node = graph.nodes.get(tag)
+        if node is None:
+            pass
+        elif node["type"] == "output":
+            raise ValueError(f"Output {tag} is generated by multiple stages: {[node['creator'], stage_name]}")
+        else:
+            raise ValueError(f"Tag {tag} is generated by pipeline stage {stage_name} but is also listed as an overall input to the pipeline")
+        outputs.append((tag, tag1, ftype))
+
+    invalidate_static_ordering(graph)
+    graph.add_node(
+        stage_name,
+        type="stage",
+        stage_class=stage_class,
+        aliases=aliases,
+        status=StageStatus.PENDING,
+        **attrs,
+    )
+    for tag, tag1, ftype in outputs:
+        graph.add_node(tag, type="output", creator=stage_name, original_tag=tag1, ftype=ftype)
+        graph.add_edge(stage_name, tag)
+
+
+def add_input_node(graph, tag, **attrs):
+    """
+    Add (or update) an overall input file in a pipeline graph.
 
-    Returns
-    -------
+    Parameters
+    ----------
     graph : `networkx.DiGraph`
-        A directed graph representing the pipeline stages, their inputs, and outputs
+        The graph to add to
+    tag : `str`
+        The tag for the file
+    **attrs
+        Any additional attributes, e.g. path, to set on the node
     """
+    node = graph.nodes.get(tag)
+    if node is not None and node["type"] != "input":
+        creator = node.get("creator", tag)
+        raise ValueError(f"Tag {tag} is generated by pipeline stage {creator} but is also listed as an overall input to the pipeline")
+    graph.add_node(tag, type="input", **attrs)
 
-    # Build a graph with all the stages and files in it
-    graph = networkx.DiGraph()
 
-    # First add all the overall inputs
-    for inp in overall_inputs:
-        graph.add_node(inp, type="input")
-    
-    # Now go through all the stages. We do this in two passes,
-    # one to add the stages themselves and their outputs, and one
-    # to connect in all their inputs
-    for stage_name, stage_class, aliases in zip(stage_names, stage_classes, stage_aliases):
-        if stage_name in graph.nodes:
-            raise ValueError(f"Stage {stage_name} is used multiple times in the pipeline")
-        graph.add_node(stage_name, type="stage", stage_class=stage_class)
-
-        # Now set up all the outputs of the stage, first checking to see if it has been aliased
-        for tag1 in stage_class.output_tags():
-            tag = aliases.get(tag1, tag1)
-            if tag in overall_inputs:
-                raise ValueError(f"Tag {tag} is generated by pipeline stage {stage_name} but is also listed as an overall input to the pipeline")
-            graph.add_node(tag, type="output", creator=stage_class, original_tag=tag1)
-            graph.add_edge(stage_name, tag)
-
-    # Now check for outputs that are generated by multiple stages
-    for node in graph.nodes:
-        if graph.nodes[node]["type"] == "output":
-            parents = list(graph.predecessors(node))
-            if len(parents) > 1:
-                raise ValueError(f"Output {node} is generated by multiple stages: {parents}")
-            
-
-    # Now go through and add in all the inputs needed for each stage:
+def remove_stage_node(graph, stage_name):
+    """
+    Remove a stage and all the outputs it generates from a pipeline graph.
+
+    Parameters
+    ----------
+    graph : `networkx.DiGraph`
+        The graph to modify
+    stage_name : `str`
+        The name of the stage
+    """
+    outputs = [n for n in graph.successors(stage_name) if graph.nodes[n]["type"] == "output"]
+    invalidate_static_ordering(graph)
+    graph.remove_nodes_from(outputs + [stage_name])
+
+
+def stage_nodes(graph):
+    """Return the names of the stage nodes in a graph, in insertion order"""
+    return [n for n, t in graph.nodes(data="type") if t == "stage"]
+
+
+def connect_stage_inputs(graph):
+    """
+    Connect the input files of each stage in the graph to that stage,
+    and check that the result is a valid pipeline, with no missing
+    inputs or cycles.
+
+    This can safely be called more than once.
+
+    Parameters
+    ----------
+    graph : `networkx.DiGraph`
+        The graph to modify
+    """
+    invalidate_static_ordering(graph)
     missing = collections.defaultdict(list)
-    for stage_name, stage_class, aliases in zip(stage_names, stage_classes, stage_aliases):
-        for tag in stage_class.input_tags():
+    for stage_name in stage_nodes(graph):
+        node = graph.nodes[stage_name]
+        aliases = node["aliases"]
+        for tag in node["stage_class"].input_tags():
             tag = aliases.get(tag, tag)
             if tag not in graph.nodes:
                 missing[tag].append(stage_name)
             else:
                 graph.add_edge(tag, stage_name)
-    
+
     if missing:
         msg = []
         for tag, stages in missing.items():
@@ -74,27 +159,79 @@ def build_graph(stage_names, stage_classes, stage_aliases, overall_inputs):
         msg = "\n".join(msg)
         raise ValueError(f"The following files are needed by the pipeline but not generated by any stage:\n{msg}")
 
-    # Now check for cycles in the graph
-    cycles = list(networkx.simple_cycles(graph))
-    if cycles:
+    # Now check for cycles in the graph, only listing them if there are any
+    if not networkx.is_directed_acyclic_graph(graph):
         msg = []
-        for cycle in cycles:
+        for cycle in networkx.simple_cycles(graph):
             cycle_text = " -> ".join(cycle)
             cycle_text += " -> " + cycle[0]
             msg.append(f"- {cycle_text}")
+        msg = "\n".join(msg)
         raise ValueError(f"Your pipeline is cyclic. The following cycles were found:\n{msg}")
-    
+
+
+def build_graph(stage_names, stage_classes, stage_aliases, overall_inputs):
+    """
+    Build a directed graph of the pipeline stages and their inputs and outputs.
+
+    This also checks for various pipeline problems like missing inputs,
+    multiple stages generating the same output, and cyclic dependencies.
+
+    Parameters
+    ----------
+    stage_names : `list` of `str`
+        The names of the stages in the pipeline
+    stage_classes : `list` of `PipelineStage` subclasses
+        The classes of the stages in the pipeline
+    stage_aliases : `list` of `dict`
+        A list of dictionaries mapping tags to aliases for each stage
+    overall_inputs : `dict` of `str: str`
+        A list of overall input files that are not generated by any stage but are needed by the pipeline
+
+    Returns
+    -------
+    graph : `networkx.DiGraph`
+        A directed graph representing the pipeline stages, their inputs, and outputs
+    """
+    graph = networkx.DiGraph()
+    for inp in overall_inputs:
+        add_input_node(graph, inp)
+    for stage_name, stage_class, aliases in zip(stage_names, stage_classes, stage_aliases):
+        add_stage_node(graph, stage_name, stage_class, aliases)
+    connect_stage_inputs(graph)
     return graph
 
+
 def get_static_ordering(graph):
-    # get a topological ordering of the graph
-    # this will be a list of all the nodes in the graph, including the input and output files
-    ordering = list(networkx.topological_sort(graph))
+    """
+    Get an order in which the stages in a pipeline graph can be run.
 
-    # cut down to just the stages
-    ordering = [node for node in ordering if graph.nodes[node]["type"] == "stage"]
-   
-    return ordering
+    The result is cached in the graph attributes, and re-used until
+    it is invalidated by a change to the graph.
+
+    Parameters
+    ----------
+    graph : `networkx.DiGraph`
+        The pipeline graph
+
+    Returns
+    -------
+    ordering : `list` of `str`
+        The names of the stages in a valid execution order
+    """
+    ordering = graph.graph.get(STATIC_ORDERING)
+
+    if ordering is None:
+        # get a topological ordering of the graph
+        # this will be a list of all the nodes in the graph, including the input and output files
+        ordering = list(networkx.topological_sort(graph))
+
+        # cut down to just the stages
+        ordering = [node for node in ordering if graph.nodes[node]["type"] == "stage"]
+        graph.graph[STATIC_ORDERING] = ordering
+
+    # return a copy so callers cannot modify the cached version
+    return list(ordering)
 
 def trim_pipeline_graph(graph, from_=None, to_=None):
     """
@@ -170,30 +307,23 @@ def trim_pipeline_graph(graph, from_=None, to_=None):
     # Create a subgraph with just these nodes
     trimmed_graph = graph.subgraph(nodes_to_keep).copy()
 
+    # The copy includes the graph attributes, so any cached
+    # ordering refers to the untrimmed graph
+    invalidate_static_ordering(trimmed_graph)
+
     # Some stages that were previously outputs of existing stages
     # must now be overall inputs to the pipeline instead because
     # the stage that generated them has been trimmed. We need to keep
     # track of this
     converted_inputs = {}
-    for node_name in list(trimmed_graph.nodes):
-        node = trimmed_graph.nodes[node_name]
+    for node_name, node in trimmed_graph.nodes(data=True):
 
         if node["type"] == "output":
             # The node will be listed as an output but will have no
             # predecessor creating it now.
-            predecessors = list(trimmed_graph.predecessors(node_name))
-            if not predecessors:
-                creator_stage_class = node["creator"]
-                original_tag = node["original_tag"]
-                # use the original tag to find the file type object
-                for t, typ in creator_stage_class.outputs:
-                    if t == original_tag:
-                        ftype = typ
-                        break
-                else:
-                    raise RuntimeError("This should never happen - please open an issue")
+            if trimmed_graph.in_degree(node_name) == 0:
                 # the node_name is the aliased_tag, so this file name should be correct
-                file_name = ftype.make_name(node_name)
+                file_name = node["ftype"].make_name(node_name)
                 converted_inputs[node_name] = file_name
 
     return trimmed_graph, converted_inputs

{ceci-2.6 → ceci-2.7}/ceci/pipeline/mini.py

@@ -1,6 +1,7 @@
 import sys
 from .. import minirunner
 from .pipeline import Pipeline, get_default_site
+from .graph import StageStatus
 
 class MiniPipeline(Pipeline):
     """A pipeline subclass that uses Minirunner, a sub-module
@@ -50,42 +51,49 @@ class MiniPipeline(Pipeline):
         Parameters
         ----------
 
-        stages: list[PipelineStage]
-            A list of stages to generate the DAG for
+        jobs: dict[str, minirunner.Job]
+            The jobs to generate the DAG for, keyed by stage name
 
         """
         depend = {}
 
         # for each stage in our pipeline ...
-        for stage in self.stages[:]:
-            if stage.instance_name not in jobs:
+        for stage_name in self.stage_names:
+            if stage_name not in jobs:
                 continue
-            job = jobs[stage.instance_name]
+            job = jobs[stage_name]
             depend[job] = []
-            # check for each of the inputs for that stage ...
-            for tag in stage.input_tags():
-                aliased_tag = stage.get_aliased_tag(tag)
-                for potential_parent in self.stages[:]:
-                    # if that stage is supplied by another pipeline stage
-                    if potential_parent.instance_name not in jobs:  # pragma: no cover
-                        continue
-                    potential_parent_tags = [
-                        potential_parent.get_aliased_tag(tag_)
-                        for tag_ in potential_parent.output_tags()
-                    ]
-                    if aliased_tag in potential_parent_tags:
-                        depend[job].append(jobs[potential_parent.instance_name])
+            # the stages it depends on are the ones that make
+            # its input files.  Stages that are not being run
+            # are not included
+            for tag in self.graph.predecessors(stage_name):
+                for parent in self.graph.predecessors(tag):
+                    if parent in jobs:
+                        depend[job].append(jobs[parent])
         return depend
 
+    def _status_callback(self, event_type, event_info):
+        """Record job events as stage status in the graph, and
+        then pass them on to any user callback."""
+        status = {
+            minirunner.EVENT_LAUNCH: StageStatus.RUNNING,
+            minirunner.EVENT_COMPLETED: StageStatus.COMPLETE,
+            minirunner.EVENT_FAIL: StageStatus.FAILED,
+        }.get(event_type)
+        if status is not None:
+            self.set_stage_status(event_info["job"].name, status)
+        if self.callback is not None:
+            self.callback(event_type, event_info)
+
     def initiate_run(self, overall_inputs):
         jobs = {}
         stages = []
         return jobs, stages
 
-    def enqueue_job(self, stage, pipeline_files):
-        sec = self.stage_execution_config[stage.instance_name]
+    def enqueue_job(self, stage, input_paths):
+        sec = self.get_stage_execution_config(stage.instance_name)
         outputs = stage.find_outputs(self.run_config["output_dir"])
-        cmd = sec.generate_full_command(pipeline_files, outputs, self.stages_config)
+        cmd = sec.generate_full_command(input_paths, outputs, self.stages_config)
         job = minirunner.Job(
             stage.instance_name,
             cmd,
@@ -106,12 +114,12 @@ class MiniPipeline(Pipeline):
         # This pipeline, being mainly for testing, can only
         # run at a single site, so we can assume here that the
         # sites are all the same
-        sec = self.stage_execution_config[self.stage_names[0]]
+        sec = self.get_stage_execution_config(self.stage_names[0])
         nodes = sec.site.info["nodes"]
 
         # Run under minirununer
         runner = minirunner.Runner(
-            nodes, graph, log_dir, callback=self.callback, sleep=self.sleep
+            nodes, graph, log_dir, callback=self._status_callback, sleep=self.sleep
         )
         interval = self.launcher_config.get("interval", 3)
         try:

{ceci-2.6 → ceci-2.7}/ceci/pipeline/parsl.py

@@ -2,6 +2,7 @@ import sys
 import os
 
 from .pipeline import Pipeline
+from .graph import StageStatus
 
 
 class ParslPipeline(Pipeline):
@@ -10,7 +11,7 @@ class ParslPipeline(Pipeline):
     def initiate_run(self, overall_inputs):
         return []  # list of futures
 
-    def enqueue_job(self, stage, pipeline_files):
+    def enqueue_job(self, stage, input_paths):
 
         from parsl.data_provider.files import File
 
@@ -22,7 +23,7 @@ class ParslPipeline(Pipeline):
         # parsl wants.
         # The inputs that exist already need to be converted into Parsl File objects.
         # The ones that don't stay as data futures
-        inputs1 = stage.find_inputs(pipeline_files)
+        inputs1 = stage.find_inputs(input_paths)
         inputs = [
             File(val) if isinstance(val, str) else val for val in inputs1.values()
         ]
@@ -47,8 +48,10 @@ class ParslPipeline(Pipeline):
             try:
                 # This waits for b/g pipeline completion.
                 future.result()
+                self.set_stage_status(stage_name, StageStatus.COMPLETE)
             # Parsl emits this on any non-zero status code.
             except BashExitFailure:
+                self.set_stage_status(stage_name, StageStatus.FAILED)
                 stdout_file = f"{log_dir}/{stage_name}.err"
                 stderr_file = f"{log_dir}/{stage_name}.out"
                 sys.stderr.write(
@@ -138,7 +141,7 @@ Standard error:
         config = f"{{inputs[{config_index}]}}"
 
         # This includes all the "mpirun" stuff.
-        sec = self.stage_execution_config[stage.instance_name]
+        sec = self.get_stage_execution_config(stage.instance_name)
         executor = sec.site.info["executor"]
 
         # Construct the command line call