ceci 2.0.1__tar.gz → 2.2__tar.gz

{ceci-2.0.1 → ceci-2.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ceci
-Version: 2.0.1
+Version: 2.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.0.1 → ceci-2.2}/ceci/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.0.1'
-__version_tuple__ = version_tuple = (2, 0, 1)
+__version__ = version = '2.2'
+__version_tuple__ = version_tuple = (2, 2)

{ceci-2.0.1 → ceci-2.2}/ceci/config.py

@@ -88,6 +88,10 @@ class StageParameter:
             self._required = self._default is None
         self._value = cast_value(self._dtype, self._default)
 
+    def __repr__(self):
+        req_str = 'required' if self.required else 'optional'
+        return f"Parameter({self.msg}, type: {self.dtype}, default: {self.default} [{req_str}])"
+
     @property
     def value(self):
         """Return the value"""
@@ -138,6 +142,27 @@ class StageParameter:
         self._default = default
         self.set_to_default()
 
+    def numpy_style_help_text(self):
+        """Create a docstring followwing numpy style guidelines"""
+        s = ""
+        if self._dtype is None:
+            s += "[type not specified] "
+        else:
+            s += f"[{self._dtype.__name__}] "
+        if self._required:
+            s += "(required)"
+        else:
+            # truncate long dicts
+            if isinstance(self._default, dict) and len(self._default) > 10:
+                s += "(default={...})"
+            # truncate long lists
+            elif isinstance(self._default, list) and len(self._default) > 10:
+                s += "(default=[...])"
+            else:
+                s += f"default={self._default}"
+        s += f"\n    {self._help}"
+        return s
+
 
 class StageConfig(dict):
     """A small class to manage a dictionary of configuration parameters with basic type checking"""
@@ -279,3 +304,19 @@ class StageConfig(dict):
         if isinstance(attr, StageParameter):
             return attr.dtype
         return type(attr)
+
+    def numpy_style_help_text(self):
+        """Create a docstring followwing numpy style guidelines"""
+        if not self:
+            return "<This class has no configuration options>"
+        s = []
+        for key, val in dict.items(self):
+            if isinstance(val, StageConfig):
+                val = val[key]
+            if isinstance(val, StageParameter):
+                s.append(f"{key}: {val.numpy_style_help_text()}")
+            elif isinstance(val, type):
+                s.append(f"{key}: {val.__name__}] (required)")
+            else:
+                s.append(f"{key}: {type(val).__name__}] (default={val})")
+        return '\n\n'.join(s)

{ceci-2.0.1 → ceci-2.2}/ceci/minirunner.py

@@ -36,9 +36,10 @@ class TimeOut(RunnerError):
 class FailedJob(RunnerError):
     """Error for when a job has failed."""
 
-    def __init__(self, msg, job_name):
+    def __init__(self, msg, job_name, run_time):
         super().__init__(msg)
         self.job_name = job_name
+        self.run_time = run_time
 
 
 class Node:
@@ -145,6 +146,21 @@ def null_callback(
 ):  # pylint: disable=unused-argument,missing-function-docstring
     pass
 
+def make_run_time_string(seconds):
+    """Convert the time in seconds to a string of the form
+    "X days, Y hours, Z minutes, W seconds"
+    """
+    if seconds < 60:
+        return f"{seconds:.1f} seconds"
+    minutes, seconds = divmod(seconds, 60)
+    if minutes < 60:
+        return f"{minutes:.0f} minutes, {seconds:.1f} seconds"
+    hours, minutes = divmod(minutes, 60)
+    if hours < 24:
+        return f"{hours:.0f} hours, {minutes:.0f} minutes, {seconds:.1f} seconds"
+    days, hours = divmod(hours, 24)
+    return f"{days:.0f} days, {hours:.0f} hours, {minutes:.0f} minutes, {seconds:.1f} seconds"
+
 
 class Runner:
     """The main pipeline runner class.
@@ -245,7 +261,7 @@ class Runner:
 
     def abort(self):
         """End the pipeline and kill all running jobs."""
-        for process, _, _ in self.running:
+        for process, _, _, _ in self.running:
             process.kill()
 
         for node in self.nodes:
@@ -275,7 +291,7 @@ class Runner:
             p = subprocess.Popen(
                 cmd, shell=True, stdout=stdout, stderr=subprocess.STDOUT
             )  # pylint: disable=consider-using-with
-            self.running.append((p, job, alloc))
+            self.running.append((p, job, alloc, default_timer()))
             self.callback(
                 EVENT_LAUNCH,
                 {"job": job, "stdout": stdout_file, "process": p, "nodes": alloc},
@@ -337,12 +353,12 @@ class Runner:
         completed_jobs = []
         continuing_jobs = []
         # loop through all known running ones
-        for process, job, alloc in self.running:
+        for process, job, alloc, start_time in self.running:
             # check status
             status = process.poll()
             # None indicates job is still running
             if status is None:  # pragma: no cover
-                continuing_jobs.append((process, job, alloc))
+                continuing_jobs.append((process, job, alloc, start_time))
             # status !=0 indicates error in job.
             # kill everything
             elif status:
@@ -355,11 +371,13 @@ class Runner:
                     EVENT_FAIL,
                     {"job": job, "status": status, "process": process, "nodes": alloc},
                 )
+                fail_time = make_run_time_string(default_timer() - start_time)
                 self.abort()
-                raise FailedJob(job.cmd, job.name)
+                raise FailedJob(job.cmd, job.name, fail_time)
             # status==0 indicates sucess in job, so free resources
             else:
-                print(f"Job {job.name} has completed successfully!")
+                run_time = make_run_time_string(default_timer() - start_time)
+                print(f"Job {job.name} has completed successfully in {run_time} seconds !")
                 # Call back with info about the successful job and how it ran
                 self.callback(
                     EVENT_COMPLETED,

{ceci-2.0.1 → ceci-2.2}/ceci/pipeline.py

@@ -6,13 +6,18 @@ import graphlib
 import yaml
 import shutil
 from abc import abstractmethod
-import warnings
+import collections
 
 from .stage import PipelineStage
 from . import minirunner
 from .sites import load, get_default_site
 from .utils import embolden, extra_paths
 
+RESUME_MODE_RESUME = "resume"
+RESUME_MODE_RESTART = "restart"
+RESUME_MODE_REFUSE = "refuse"
+
+
 
 def override_config(config, extra):
     """Override configuration parameters with extra parameters provided on command line
@@ -364,11 +369,16 @@ class Pipeline:
         run_config = {
             "output_dir": pipe_config.get("output_dir", "."),
             "log_dir": pipe_config.get("log_dir", "."),
-            "resume": pipe_config.get("resume", False),
+            "resume": pipe_config.get("resume", RESUME_MODE_RESUME),
             "flow_chart": pipe_config.get("flow_chart", ""),
             "registry": pipe_config.get("registry", None),
         }
 
+        if run_config["resume"] is True:
+            run_config["resume"] = RESUME_MODE_RESUME
+        elif run_config["resume"] is False:
+            run_config["resume"] = RESUME_MODE_RESTART
+
         launcher_dict = dict(cwl=CWLPipeline, parsl=ParslPipeline, mini=MiniPipeline)
 
         if pipe_config.get("flow_chart", False):
@@ -514,18 +524,23 @@ class Pipeline:
         else:
             raise ValueError("Must specify either id or name in registry lookup")
 
+        # Whatever the lookup, we always require a dataset which has not beed deleted or replaced
+        status_filter = registry.Query.gen_filter("dataset.status", "==", 1)
+
         # Main finder method
-        results = registry.Query.find_datasets(["dataset.dataset_id"], [filter])
+        results = registry.Query.find_datasets(["dataset.dataset_id"], [filter, status_filter])
 
         # Check that we find exactly one dataset matching the query
-        results = list(results)
-        if len(results) == 0:
+        if not results:
             raise ValueError(f"Could not find any dataset matching {info} in registry")
-        elif len(results) > 1:
+
+        results = results['dataset.dataset_id']
+
+        if len(results) > 1:
             raise ValueError(f"Found multiple datasets matching {info} in registry")
 
         # Get the absolute path
-        return registry.Query.get_dataset_absolute_path(results[0].dataset_id)
+        return registry.Query.get_dataset_absolute_path(results[0])
 
 
     def process_overall_inputs(self, inputs):
@@ -820,7 +835,9 @@ class Pipeline:
             # In interactive use the stage object might already be created
             # otherwise we need to create it
             if sec.stage_obj is not None:
+                orig_stage_config = stages_config_data.get(stage_name, {}).copy()
                 stage = sec.stage_obj
+                stage.config.update(**orig_stage_config)
             else:
                 # Find the inputs for this stage and set up the arguments
                 # to the stage init method
@@ -949,7 +966,24 @@ class Pipeline:
 
     def should_skip_stage(self, stage):
         """Return true if we should skip a stage because it is finished and we are in resume mode"""
-        return stage.should_skip(self.run_config)
+        resume_mode = self.run_config["resume"]
+        if resume_mode is True:
+            resume_mode = "resume"
+        elif resume_mode is False:
+            resume_mode = "restart"
+
+        if resume_mode == "restart":
+            return False
+
+        outputs = stage.find_outputs(self.run_config["output_dir"]).values()
+
+        if resume_mode == "resume":
+            return all(os.path.exists(output) for output in outputs)
+        elif resume_mode == "refuse":
+            if any(os.path.exists(output) for output in outputs):
+                raise RuntimeError(f"Output files already exist for stage {stage} and resume mode is 'refuse'")
+        else:
+            raise ValueError(f"Unknown resume mode: {resume_mode}")
 
     def save(self, pipefile, stagefile=None, reduce_config=False, **kwargs):
         """Save this pipeline state to a yaml file
@@ -1039,6 +1073,10 @@ class Pipeline:
         # Nodes we have already added
         seen = set()
 
+        # Dictionary to track nodes by their inputs and outputs
+        node_groups = {}
+
+
         # Add overall pipeline inputs
         for inp in self.overall_inputs.keys():
             graph.add_node(inp, shape="box", color="gold", style="filled")
@@ -1064,6 +1102,43 @@ class Pipeline:
                     seen.add(out)
                 graph.add_edge(stage.instance_name, out, color="black")
 
+        # We want to group together all the files that all created
+        # by the same stage and also all used by the same stages, to
+        # reduce the number of nodes in the graph and make it more readable.
+        # First we find that grouping.
+        node_groups = collections.defaultdict(list)
+        for node in graph.nodes_iter():
+            # only affect the nodes representing files
+            if node.attr['color'] != "skyblue":
+                continue
+            # Find the stage node that created this file,
+            # and all the stage nodes that make use of it
+            edge_in = graph.in_edges(node)[0]
+            creator = edge_in[0]
+            users = []
+            for edge in graph.out_edges(node):
+                users.append(edge[1])
+            key = (creator, tuple(users))
+            node_groups[key].append(node)
+
+        # Now we remove all the groups of nodes with more than one in
+        # and replace them with a single node
+        for key, nodes in node_groups.items():
+            if len(nodes) > 1:
+                if len(nodes) > 4:
+                    # make a string with two nodes per line
+                    node_names = []
+                    for i in range(0, len(nodes), 2):
+                        node_names.append(",  ".join(nodes[i:i+2]))
+                    new_node = "\n".join(node_names)
+                else:
+                    new_node = "\n".join(nodes)
+                graph.remove_nodes_from(nodes)
+                graph.add_node(new_node, shape="box", color="skyblue", style="filled")
+                graph.add_edge(key[0], new_node, color="black")
+                for user in key[1]:
+                    graph.add_edge(new_node, user, color="black")
+
         # finally, output the stage to file
         if filename.endswith(".dot"):
             graph.write(filename)
@@ -1432,6 +1507,7 @@ class MiniPipeline(Pipeline):
                 f"""
 *************************************************
 Error running pipeline stage {error.job_name}.
+Failed after {error.run_time}.
 
 Standard output and error streams in {log_dir}/{error.job_name}.out
 *************************************************

{ceci-2.0.1 → ceci-2.2}/ceci/stage.py

@@ -9,11 +9,13 @@ import cProfile
 import pdb
 import datetime
 import warnings
+import socket
 
 from abc import abstractmethod
 from . import errors
 from .monitor import MemoryMonitor
 from .config import StageParameter, StageConfig, cast_to_streamable
+from .utils import activate_tracing
 
 SERIAL = "serial"
 MPI_PARALLEL = "mpi"
@@ -146,9 +148,19 @@ class PipelineStage:
 
     @abstractmethod
     def run(self):  # pragma: no cover
-        """Run the stage and return the execution status"""
+        """Run the stage and return the execution status.
+
+        Subclasses must implemented this method.
+        """
         raise NotImplementedError("run")
 
+    def validate(self):
+        """Check that the inputs actually have the data needed for execution,
+        This is called before the run method. It is an optional stage, meant
+        for checking that the input to the stage is actual in the form and
+        shape needed before an expensive run is executed."""
+        pass
+
     def load_configs(self, args):
         """
         Load the configuraiton
@@ -340,14 +352,15 @@ class PipelineStage:
         path = pathlib.Path(filename).resolve()
 
         # Add a description of the parameters to the end of the docstring
-        if stage_is_complete:
+        # If no config options are specified, omit this.
+        if stage_is_complete and cls.config_options:
             config_text = cls._describe_configuration_text()
             if cls.__doc__ is None:
-                cls.__doc__ = f"Stage {cls.name}\n\nConfiguration Parameters:\n{config_text}"
+                cls.__doc__ = f"Stage {cls.name}\n\nParameters\n----------\n{config_text}"
             else:
                 # strip any existing configuration text from parent classes that is at the end of the doctring
-                cls.__doc__ = cls.__doc__.split("Configuration Parameters:")[0]
-                cls.__doc__ += f"\n\nConfiguration Parameters:\n{config_text}"
+                cls.__doc__ = cls.__doc__.split("Parameters")[0]
+                cls.__doc__ += f"\n\nParameters\n----------\n{config_text}"
 
         # Register the class
         if stage_is_complete:
@@ -423,24 +436,24 @@ class PipelineStage:
         s = []
         if cls.config_options is None:
             return "<This class has no configuration options>"
+
         for name, val in cls.config_options.items():
+            if isinstance(val, StageConfig):
+                val = val[name]
             if isinstance(val, StageParameter):
-                if val.required:
-                    if val.dtype is None:
-                        txt = f"[type not specified]: {val._help} (required)"
-                    else:
-                        txt = f"[{val.dtype.__name__}]: {val._help}  (required)"
-                else:
-                    if val.dtype is None:
-                        txt = f"[type not specified]: {val._help} (default={val.default})"
-                    else:
-                        txt = f"[{val.dtype.__name__}]: {val._help} (default={val.default})"
+                s.append(f"{name}: {val.numpy_style_help_text()}")
             elif isinstance(val, type):
-                txt = f"[{val.__name__}]: (required)"
+                s.append(f"{name}: {val.__name__}] (required)")
             else:
-                txt = f"[{type(val).__name__}]: (default={val})"
-            s.append(f"{name} {txt} ")
-        return '\n'.join(s)
+                s.append(f"{name}: {type(val).__name__}] (default={val})")
+
+        for input_ in cls.inputs:
+            s.append(f"{input_[0]}: {input_[1].__name__} (INPUT)")
+        for output_ in cls.outputs:
+            s.append(f"{output_[0]}: {output_[1].__name__} (OUTPUT)")
+
+        return '\n\n'.join(s)
+
 
     @classmethod
     def usage(cls):  # pragma: no cover
@@ -603,6 +616,12 @@ I currently know about these stages:
             help="Report memory use. Argument gives interval in seconds between reports",
         )
 
+        parser.add_argument(
+            "--trace",
+            action="store_true",
+            help="Enable sending a signal to the process that prints a trace wherever it is",
+        )
+
         # Error message we will return if --mpi used on a non-supported
         # stage.
         mpi_err = (
@@ -661,6 +680,21 @@ I currently know about these stages:
         if args.memmon:  # pragma: no cover
             monitor = MemoryMonitor.start_in_thread(interval=args.memmon)
 
+        if args.trace:
+            activate_tracing(stage._rank)
+
+        # Now we try to see if the validation step has been changed,
+        # if it has then we will run the validation step, and raise any errors
+        try:
+            stage.validate()
+        except Exception as error:
+            if stage.rank==0:
+                print(f"Looks like there is an validation error in: {cls.name}",
+                        "the input data for this stage did not pass the checks implemented on it.")
+                print(error)
+            raise
+
+
         try:
             stage.run()
         except Exception as error:  # pragma: no cover
@@ -781,13 +815,24 @@ I currently know about these stages:
 
     def is_mpi(self):
         """
-        Returns True if the stage is being run under MPI.
+        Check if the stage is being run under MPI.
+
+        Returns
+        -------
+        bool
+            True if the stage is being run under MPI
         """
         return self._parallel == MPI_PARALLEL
 
     def is_dask(self):
         """
-        Returns True if the stage is being run in parallel with Dask.
+        Check if the stage is being run in parallel with Dask.
+
+        Returns
+        -------
+        bool
+            True if the stage is being run under MPI
+        
         """
         return self._parallel == DASK_PARALLEL
 
@@ -938,6 +983,11 @@ I currently know about these stages:
         Parallel: bool
             Whether to split data by rank or just give all procs all data.
             Default=True
+
+        Returns
+        -------
+        start, end: tuple
+            The start and end of the range of rows to be read by this process    
         """
         n_chunks = n_rows // chunk_rows
         if n_chunks * chunk_rows < n_rows:  # pragma: no cover
@@ -959,6 +1009,17 @@ I currently know about these stages:
         """
         Return the path of an input file with the given tag,
         which can be aliased.
+
+        Parameters
+        ----------
+        tag: str
+            Tag as listed in self.outputs
+
+        Returns
+        -------
+        path: str
+            The path to the output file
+        
         """
         tag = self.get_aliased_tag(tag)
         return self._inputs[tag]
@@ -971,7 +1032,21 @@ I currently know about these stages:
         which can be aliased already.
 
         If final_name is False then use a temporary name - file will
-        be moved to its final name at the end
+        be moved to its final name at the end. The temporary name
+        is prefixed with `inprogress_`.
+
+        Parameters
+        ----------
+        tag: str
+            Tag as listed in self.outputs
+
+        final_name: bool
+            Default=False. Whether to save to the final name.
+
+        Returns
+        -------
+        path: str
+            The path to the output file
         """
 
         tag = self.get_aliased_tag(tag)
@@ -994,6 +1069,21 @@ I currently know about these stages:
         For specialized file types like FITS or HDF5 it will return
         a more specific object - see the types.py file for more info.
 
+        Parameters
+        ----------
+        tag: str
+            Tag as listed in self.inputs
+
+        wrapper: bool
+            Whether to return an underlying file object (False) or a data type instance (True)
+
+        **kwargs: dict
+            Extra arguments to pass to the file class constructor
+
+        Returns
+        -------
+        obj: file or object
+            The opened file or object
         """
         path = self.get_input(tag)
         input_class = self.get_input_type(tag)
@@ -1010,7 +1100,7 @@ I currently know about these stages:
         Find and open an output file with the given tag, in write mode.
 
         If final_name is True then they will be opened using their final
-        target output name.  Otherwise we will prepend "inprogress_" to their
+        target output name.  Otherwise we will prepend `inprogress_` to their
         file name. This means we know that if the final file exists then it
         is completed.
 
@@ -1021,12 +1111,11 @@ I currently know about these stages:
 
         Parameters
         ----------
-
         tag: str
             Tag as listed in self.outputs
 
         wrapper: bool
-            Default=False.  Whether to return a wrapped file
+            Whether to return an underlying file object (False) or a data type instance (True)
 
         final_name: bool
             Default=False. Whether to save to
@@ -1034,6 +1123,10 @@ I currently know about these stages:
         **kwargs:
             Extra args are passed on to the file's class constructor.
 
+        Returns
+        -------
+        obj: file or object
+            The opened file or object
         """
         path = self.get_output(tag, final_name=final_name)
         output_class = self.get_output_type(tag)
@@ -1078,33 +1171,63 @@ I currently know about these stages:
     @classmethod
     def inputs_(cls):
         """
-        Return the dict of inputs
+        Return the dict mapping input tags to file names.
+
+        Returns
+        -------
+        in_dict : dict[str:str]
         """
         return cls.inputs  # pylint: disable=no-member
 
     @classmethod
     def outputs_(cls):
         """
-        Return the dict of inputs
+        Return the dict mapping output tags to file names.
+
+        Returns
+        -------
+        out_dict : dict[str:str]
         """
         return cls.outputs  # pylint: disable=no-member
 
     @classmethod
     def output_tags(cls):
         """
-        Return the list of output tags required by this stage
+        Return the list of output tags required by this stage.
+
+        Returns
+        -------
+        out_tags : list[str]
+            The list of output tags
         """
         return [tag for tag, _ in cls.outputs_()]
 
     @classmethod
     def input_tags(cls):
         """
-        Return the list of input tags required by this stage
+        Return the list of input tags required by this stage.
+
+        Returns
+        -------
+        in_tags : list[str]
+            The list of input tags
         """
         return [tag for tag, _ in cls.inputs_()]
 
     def get_input_type(self, tag):
-        """Return the file type class of an input file with the given tag."""
+        """
+        Return the file type class of an input file with the given tag.
+
+        Parameters
+        ----------
+        tag : str
+            The tag of the input file
+
+        Returns
+        -------
+        ftype : FileType
+            The file type class
+        """
         tag = self.get_aliased_tag(tag)
         for t, dt in self.inputs_():
             t = self.get_aliased_tag(t)
@@ -1113,7 +1236,19 @@ I currently know about these stages:
         raise ValueError(f"Tag {tag} is not a known input")  # pragma: no cover
 
     def get_output_type(self, tag):
-        """Return the file type class of an output file with the given tag."""
+        """
+        Return the file type class of an output file with the given tag.
+
+        Parameters
+        ----------
+        tag : str
+            The tag of the output file
+
+        Returns
+        -------
+        ftype : FileType
+            The file type class
+        """
         tag = self.get_aliased_tag(tag)
         for t, dt in self.outputs_():
             t = self.get_aliased_tag(t)
@@ -1133,8 +1268,12 @@ I currently know about these stages:
     @property
     def config(self):
         """
-        Returns the configuration dictionary for this stage, aggregating command
+        The configuration dictionary for this stage, aggregating command
         line options and optional configuration file.
+
+        Options specified in the subclass variable `config_options` are
+        read from the configuration file, command line, or `make_stage` choices,
+        and stored in this dictionary.
         """
         return self._configs
 
@@ -1252,12 +1391,6 @@ I currently know about these stages:
                 f"{tag:20} : {aliased_tag:20} :{str(ftype):20} : {self._outputs[aliased_tag]}\n"
             )
 
-    def should_skip(self, run_config):
-        """Return true if we should skip a stage b/c it's outputs already exist and we are in resume mode"""
-        outputs = self.find_outputs(run_config["output_dir"]).values()
-        already_run_stage = all(os.path.exists(output) for output in outputs)
-        return already_run_stage and run_config["resume"]
-
     def already_finished(self):
         """Print a warning that a stage is being skipped"""
         print(f"Skipping stage {self.instance_name} because its outputs exist already")
@@ -1269,6 +1402,7 @@ I currently know about these stages:
         Loop through chunks of the input data from a FITS file with the given tag
 
         TODO: add ceci tests of this functions
+
         Parameters
         ----------
         tag: str
@@ -1357,6 +1491,59 @@ I currently know about these stages:
             data = {col: group[col][start:end] for col in cols}
             yield start, end, data
 
+    def combined_iterators(self, rows, *inputs, parallel=True):
+        """
+        Iterate through multiple files at the same time.
+
+        If you have more several HDF files with the some
+        columns of the same length then you can use this method to
+        iterate through them all at once, and combine the data from
+        all of them into a single dictionary.
+
+        Parameters
+        ----------
+        rows: int
+            The number of rows to read in each chunk
+
+        *inputs: list
+            A list of (tag, group, cols) triples for each file to read.
+            In each case tag is the input file name tag, group is the
+            group within the HDF5 file to read, and cols is a list of
+            columns to read from that group.  Specify multiple triplets
+            to read from multiple files
+
+        parallel: bool
+            Whether to split up data among processes (parallel=True) or give
+            all processes all data (parallel=False).  Default = True.
+
+        Returns
+        -------
+        it: iterator
+            Iterator yielding (int, int, dict) tuples of (start, end, data)
+        """
+        if not len(inputs) % 3 == 0:
+            raise ValueError(
+                "Arguments to combined_iterators should be in threes: "
+                "tag, group, value"
+            )
+        n = len(inputs) // 3
+
+        iterators = []
+        for i in range(n):
+            tag = inputs[3 * i]
+            section = inputs[3 * i + 1]
+            cols = inputs[3 * i + 2]
+            iterators.append(
+                self.iterate_hdf(tag, section, cols, rows, parallel=parallel)
+            )
+
+        for it in zip(*iterators):
+            data = {}
+            for (s, e, d) in it:
+                data.update(d)
+            yield s, e, data
+
+
     ################################
     # Pipeline-related methods
     ################################
@@ -1556,3 +1743,52 @@ I currently know about these stages:
         # cwl_tool.metadata = cwlgen.Metadata(**metadata)
 
         return cwl_tool
+
+
+    def time_stamp(self, tag):
+        """
+        Print a time stamp with an optional tag.
+
+        Parameters
+        ----------
+        tag: str
+            Additional info to print in the output line. Default is empty.
+        """
+        t = datetime.datetime.now()
+        print(f"Process {self.rank}: {tag} {t}")
+        sys.stdout.flush()
+
+    def memory_report(self, tag=None):
+        """
+        Print a report about memory currently available
+        on the node the process is running on.
+
+        Parameters
+        ----------
+        tag: str
+            Additional info to print in the output line. Default is empty.
+        """
+        import psutil
+
+        t = datetime.datetime.now()
+
+        # The different types of memory are really fiddly and don't
+        # correspond to how you usually imagine. The simplest thing
+        # to report here is just how much memory is left on the machine.
+        mem = psutil.virtual_memory()
+        avail = mem.available / 1024**3
+        total = mem.total / 1024**3
+
+        if tag is None:
+            tag = ""
+        else:
+            tag = f" {tag}:"
+
+        # This gives you the name of the host.  At NERSC that is the node name
+        host = socket.gethostname()
+
+        # Print messsage
+        print(
+            f"{t}: Process {self.rank}:{tag} Remaining memory on {host} {avail:.1f} GB / {total:.1f} GB"
+        )
+        sys.stdout.flush()

{ceci-2.0.1 → ceci-2.2}/ceci/utils.py

@@ -3,6 +3,8 @@
 from contextlib import contextmanager
 import sys
 import os
+import signal
+import faulthandler
 
 
 def add_python_path(path, start):
@@ -107,3 +109,9 @@ def embolden(text):
         Emboldened text
     """
     return "\033[1m" + text + "\033[0m"
+
+
+def activate_tracing(rank):
+    pid = os.getpid()
+    print(f"Activating trace mode for process {rank}. Trigger trace using command: kill -s {signal.SIGUSR1} {pid}")
+    faulthandler.register(signal.SIGUSR1)

{ceci-2.0.1 → ceci-2.2}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ceci
-Version: 2.0.1
+Version: 2.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.0.1 → ceci-2.2}/tests/test_pipeline.py

@@ -130,7 +130,7 @@ class FailingStage(PipelineStage):
 
 
 def _return_value_test_(resume):
-    expected_status = 0 if resume else 1
+    expected_status = 0 if ((resume == "resume") or (resume is True)) else 1
     # Mini pipeline should not run
     launcher_config = {"interval": 0.5, "name": "mini"}
     run_config = {
@@ -159,10 +159,28 @@ def _return_value_test_(resume):
 
 def test_resume():
     _return_value_test_(True)
+    _return_value_test_("resume")
 
 
 def test_fail():
     _return_value_test_(False)
+    _return_value_test_("restart")
+
+def test_refuse():
+    launcher_config = {"interval": 0.5, "name": "mini"}
+    run_config = {
+        "log_dir": "./tests/logs",
+        "output_dir": "./tests/inputs",
+        "resume": "refuse",
+    }
+
+    pipeline = MiniPipeline([{"name": "FailingStage"}], launcher_config)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        pipeline.initialize({}, run_config, "tests/config.yml")
+        pipeline.run()
+
+    assert "Output files already exist for stage" in str(excinfo.value)
 
 
 def test_dry_run(mocker):

{ceci-2.0.1 → ceci-2.2}/tests/test_stage.py

@@ -5,8 +5,12 @@ import numpy as np
 from ceci.errors import *
 import pytest
 import h5py
+import sys
 import os
-
+import tempfile
+import time
+import subprocess
+import signal
 # TODO: test MPI facilities properly with:
 # https://github.com/rmjarvis/TreeCorr/blob/releases/4.1/tests/mock_mpi.py
 
@@ -145,7 +149,7 @@ def test_parameter():
     assert stage_1.config.b == 'puffins are not extinct?'
     assert 1 in stage_1.config.c
     assert len(stage_1.config.d) == 0
-    assert "c [list]: a list (default=[1, 2, 3])" in stage_1.__doc__
+    assert "[list] default=[1, 2, 3]" in stage_1.__doc__
     stage_1.describe_configuration()
     
     cmd = "TestStage", "--a", "6", "--b", "puffins are not extinct?", "--inp", "dummy"
@@ -568,7 +572,79 @@ def test_wrong_mpi_flag():
     with pytest.raises(ValueError):
         assert LimaSerial.parse_command_line(["LimaSerial", "--mpi"]).mpi
 
+@pytest.mark.skipif(os.environ.get("GITHUB_ACTIONS") == "true", reason="Doesn't work on github actions")
+def test_tracing():
+    with open("mike_stage.py", "w") as f:
+        f.write("""
+import ceci
+import time
+class Mike(ceci.PipelineStage):
+    name = "Mike"
+    inputs = []
+    outputs = []
+    config_options = {}
+
+    def run(self):
+        time.sleep(6)
+        print("Mike complete")
 
+if __name__ == "__main__":
+    ceci.PipelineStage.main()
+""")
+    with open("config.yml", "w") as f:
+        f.write("{}")
+
+    cmd = f"{sys.executable} mike_stage.py Mike --config config.yml --trace"
+    p1 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    time.sleep(1)
+    p1.send_signal(signal.SIGUSR1)
+    try:
+        outs, _ = p1.communicate(timeout=15)
+    except subprocess.TimeoutExpired:
+        p1.kill()
+        outs, _ = p1.communicate()
+        raise ValueError("Timeout expired in Mike test with outs =" + outs.decode())
+    assert 'mike_stage.py", line 15' in outs.decode()
+
+
+def test_combined_iterators():
+    class Oscar(PipelineStage):
+        inputs = [("inp1", HDFFile), ("inp2", HDFFile)]
+        outputs = []
+        config_options = {}
+        def run(self):
+            it = self.combined_iterators(
+                10,
+                "inp1", "group1", ["x"], 
+                "inp2", "group1", ["y", "z"])
+            for (s, e, data) in it:
+                pass
+        
+    oo = Oscar.make_stage(inp1="tests/test.hdf5", inp2="tests/test.hdf5")
+    oo.run()
+
+
+def test_memory_and_time_reports(capsys):
+    class November(PipelineStage):
+        name = f"November"
+        parallel = False
+        inputs = []
+        outputs = []
+        config_options = {}
+
+        def run(self):
+            self.memory_report()
+            self.memory_report("TAGTAG")
+            self.time_stamp("Hello")
+
+    
+    nn = November.make_stage()
+    nn.run()
+    captured = capsys.readouterr()
+    assert "Remaining memory on" in captured.out
+    assert "TAGTAG" in captured.out
+    assert "Process 0: Hello" in captured.out
+    
 
 if __name__ == "__main__":
     test_construct()