ceci 2.3.1__tar.gz → 2.4__tar.gz

{ceci-2.3.1 → ceci-2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.3.1
+Version: 2.4
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.3.1 → ceci-2.4}/ceci/__init__.py

@@ -9,6 +9,7 @@ from .pipeline import (
     FlowChartPipeline,
 )
 from .main import run_pipeline
+from . import file_types
 import importlib.metadata
 
 try:

{ceci-2.3.1 → ceci-2.4}/ceci/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.3.1'
-__version_tuple__ = version_tuple = (2, 3, 1)
+__version__ = version = '2.4'
+__version_tuple__ = version_tuple = (2, 4)
 
-__commit_id__ = commit_id = 'g453d12b8e'
+__commit_id__ = commit_id = 'g727f0fadf'

ceci-2.4/ceci/file_types.py

@@ -0,0 +1,496 @@
+import uuid
+import datetime
+import socket
+import getpass
+import warnings
+import pathlib
+import yaml
+import shutil
+import pickle
+from io import UnsupportedOperation
+
+
+class FileValidationError(Exception):
+    pass
+
+
+class DataFile:
+    """
+    A class representing a DataFile to be made by pipeline stages
+    and passed on to subsequent ones.
+
+    DataFile itself should not be instantiated - instead subclasses
+    should be defined for different file types.
+
+    These subclasses are used in the definition of pipeline stages
+    to indicate what kind of file is expected.  The "suffix" attribute,
+    which must be defined on subclasses, indicates the file suffix.
+
+    The open method, which can optionally be overridden, is used by the
+    machinery of the PipelineStage class to open an input our output
+    named by a tag.
+
+    """
+
+    supports_parallel_write = False
+    suffix = None
+
+    def __init__(self, path, mode, extra_provenance=None, validate=True, **kwargs):
+        self.path = path
+        self.mode = mode
+
+        if mode not in ["r", "w"]:
+            raise ValueError(f"File 'mode' argument must be 'r' or 'w' not '{mode}'")
+
+        self.file = self.open(path, mode, **kwargs)
+
+        if validate and mode == "r":
+            self.validate()
+
+        if mode == "w":
+            self.provenance = self.generate_provenance(extra_provenance)
+            self.write_provenance()
+        else:
+            self.provenance = self.read_provenance()
+
+    @staticmethod
+    def generate_provenance(extra_provenance=None):
+        """
+        Generate provenance information - a dictionary
+        of useful information about the origina
+        """
+        UUID = uuid.uuid4().hex
+        creation = datetime.datetime.now().isoformat()
+        domain = socket.getfqdn()
+        username = getpass.getuser()
+
+        # Add other provenance and related
+        provenance = {
+            "uuid": UUID,
+            "creation": creation,
+            "domain": domain,
+            "username": username,
+        }
+
+        if extra_provenance:
+            provenance.update(extra_provenance)
+        return provenance
+
+    def write_provenance(self):
+        """
+        Concrete subclasses (for which it is possible) should override
+        this method to save the dictionary self.provenance to the file.
+        """
+        pass
+
+    def add_provenance(self, key, value):
+        """
+        Concrete subclasses (for which it is possible) should override
+        this method to save the a new string key/value pair to file
+        """
+        pass
+
+    def read_provenance(self):
+        """
+        Concrete subclasses for which it is possible should override
+        this method and read the provenance information from the file.
+
+        Other classes will return this dictionary of UNKNOWNs
+        """
+        provenance = {
+            "uuid": "UNKNOWN",
+            "creation": "UNKNOWN",
+            "domain": "UNKNOWN",
+            "username": "UNKNOWN",
+        }
+
+        return provenance
+
+    def validate(self):
+        """
+        Concrete subclasses should override this method
+        to check that all expected columns are present.
+        """
+        pass
+
+    @classmethod
+    def open(cls, path, mode):
+        """
+        Open a data file.  The base implementation of this function just
+        opens and returns a standard python file object.
+
+        Subclasses can override to either open files using different openers
+        (like fitsio.FITS), or, for more specific data types, return an
+        instance of the class itself to use as an intermediary for the file.
+
+        """
+        return open(path, mode)
+
+    def close(self):
+        self.file.close()
+
+    @classmethod
+    def make_name(cls, tag):
+        if cls.suffix:
+            return f"{tag}.{cls.suffix}"
+        else:
+            return tag
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+
+class HDFFile(DataFile):
+    supports_parallel_write = True
+    """
+    A data file in the HDF5 format.
+    Using these files requires the h5py package, which in turn
+    requires an HDF5 library installation.
+
+    """
+    suffix = "hdf5"
+    required_datasets = []
+
+    @classmethod
+    def open(cls, path, mode, **kwargs):
+        # Suppress a warning that h5py always displays
+        # on import
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            import h5py
+        # Return an open h5py File
+        return h5py.File(path, mode, **kwargs)
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to an HDF5 file opened in read-only mode ({self.mode}")
+
+        # This method *must* be called by all the processes in a parallel
+        # run.
+        self._provenance_group = self.file.create_group("provenance")
+
+        # Call the sub-method to do each item
+        for key, value in self.provenance.items():
+            self._provenance_group.attrs[key] = value
+
+    def read_provenance(self):
+        try:
+            group = self.file["provenance"]
+            attrs = group.attrs
+        except KeyError:
+            group = None
+            attrs = {}
+
+        provenance = {
+            "uuid": attrs.get("uuid", "UNKNOWN"),
+            "creation": attrs.get("creation", "UNKNOWN"),
+            "domain": attrs.get("domain", "UNKNOWN"),
+            "username": attrs.get("username", "UNKNOWN"),
+        }
+
+        self._provenance_group = group
+
+        return provenance
+
+    def validate(self):
+        missing = [name for name in self.required_datasets if name not in self.file]
+        if missing:
+            text = "\n".join(missing)
+            raise FileValidationError(f"These data sets are missing from HDF file {self.path}:\n{text}")
+
+    def close(self):
+        self.file.close()
+
+
+class FitsFile(DataFile):
+    """
+    A data file in the FITS format.
+    Using these files requires the fitsio package.
+    """
+
+    suffix = "fits"
+    required_columns = []
+
+    @classmethod
+    def open(cls, path, mode, **kwargs):
+        import fitsio
+
+        # Fitsio doesn't have pure 'w' modes, just 'rw'.
+        # Maybe we should check if the file already exists here?
+        if mode == "w":
+            mode = "rw"
+        return fitsio.FITS(path, mode=mode, **kwargs)
+
+    def missing_columns(self, columns, hdu=1):
+        """
+        Check that all supplied columns exist
+        and are in the chosen HDU
+        """
+        ext = self.file[hdu]
+        found_cols = ext.get_colnames()
+        missing_columns = [col for col in columns if col not in found_cols]
+        return missing_columns
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        # Call the sub-method to do each item
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to a FITS file opened in read-only mode ({self.mode}")
+
+        for key, value in self.provenance.items():
+            if isinstance(value, str) and "\n" in value:
+                values = value.split("\n")
+                for i, v in enumerate(values):
+                    self.file[0].write_key(key + f"_{i}", v)
+            else:
+                self.file[0].write_key(key, value)
+
+    def read_provenance(self):
+        header = self.file[0].read_header()
+        provenance = {
+            "uuid": header.get("uuid", "UNKNOWN"),
+            "creation": header.get("creation", "UNKNOWN"),
+            "domain": header.get("domain", "UNKNOWN"),
+            "username": header.get("username", "UNKNOWN"),
+        }
+        return provenance
+
+    def validate(self):
+        """Check that the catalog has all the required columns and complain otherwise"""
+        # Find any columns that do not exist in the file
+        missing = self.missing_columns(self.required_columns)
+
+        # If there are any, raise an exception that lists them explicitly
+        if missing:
+            text = "\n".join(missing)
+            raise FileValidationError(f"These columns are missing from FITS file {self.path}:\n{text}")
+
+    def close(self):
+        self.file.close()
+
+
+class TextFile(DataFile):
+    """
+    A data file in plain text format.
+    """
+
+    suffix = "txt"
+
+
+class YamlFile(DataFile):
+    """
+    A data file in yaml format.
+    The top-level object in TXPipe YAML
+    files should always be a dictionary.
+    """
+
+    suffix = "yml"
+
+    def __init__(self, path, mode, extra_provenance=None, validate=True, load_mode="full"):
+        self.path = path
+        self.mode = mode
+        self.file = self.open(path, mode)
+
+        if mode == "r":
+            if load_mode == "safe":
+                self.content = yaml.safe_load(self.file)
+            elif load_mode == "full":
+                self.content = yaml.full_load(self.file)
+            elif load_mode == "unsafe":
+                self.content = yaml.unsafe_load(self.file)
+            else:
+                raise ValueError(f"Unknown value {yaml_load} of load_mode. Should be 'safe', 'full', or 'unsafe'")
+            # get provenance
+            self.provenance = self.read_provenance()
+
+        else:
+            self.provenance = self.generate_provenance(extra_provenance)
+            self.write_provenance()
+
+    def read(self, key):
+        return self.content[key]
+
+    def write(self, d):
+        if not isinstance(d, dict):
+            raise ValueError("Only dicts should be passed to YamlFile.write")
+        yaml.dump(d, self.file)
+
+    def write_provenance(self):
+        d = {"provenance": self.provenance}
+        self.write(d)
+
+    def read_provenance(self):
+        prov = self.content.pop("provenance", {})
+        req_provenance = {
+            "uuid": prov.get("uuid", "UNKNOWN"),
+            "creation": prov.get("creation", "UNKNOWN"),
+            "domain": prov.get("domain", "UNKNOWN"),
+            "username": prov.get("username", "UNKNOWN"),
+        }
+        prov.update(req_provenance)
+        return prov
+
+
+class Directory(DataFile):
+    suffix = ""
+
+    @classmethod
+    def open(self, path, mode):
+        p = pathlib.Path(path)
+
+        if mode == "w":
+            if p.exists():
+                shutil.rmtree(p)
+            p.mkdir(parents=True)
+        else:
+            if not p.is_dir():
+                raise ValueError(f"Directory input {path} does not exist")
+        return p
+
+    def close(self):
+        pass
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        # This method *must* be called by all the processes in a parallel
+        # run.
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to a directory opened in read-only mode ({self.mode})")
+
+        self._provenance_file = open(self.file / "provenance.yml", "w")
+
+        # Call the sub-method to do each item
+        yaml.dump(self.provenance, self._provenance_file)
+        self._provenance_file.flush()
+
+    def read_provenance(self):
+        try:
+            f = open(self.file / "provenance.yml")
+            attrs = yaml.load(f)
+
+        except KeyError:
+            f = None
+            attrs = {}
+
+        self._provenance_file = f
+
+        provenance = {
+            "uuid": attrs.get("uuid", "UNKNOWN"),
+            "creation": attrs.get("creation", "UNKNOWN"),
+            "domain": attrs.get("domain", "UNKNOWN"),
+            "username": attrs.get("username", "UNKNOWN"),
+        }
+
+        return provenance
+
+
+class FileCollection(Directory):
+    """
+    Represents a grouped bundle of files, for cases where you don't
+    know the exact list in advance.
+    """
+
+    suffix = ""
+
+    def write_listing(self, filenames):
+        """
+        Write a listing file in the directory recording
+        (presumably) the filenames put in it.
+        """
+        fn = self.path_for_file("txpipe_listing.txt")
+        with open(fn, "w") as f:
+            yaml.dump(filenames, f)
+
+    def read_listing(self):
+        """
+        Read a listing file from the directory.
+        """
+        fn = self.path_for_file("txpipe_listing.txt")
+        with open(fn, "w") as f:
+            filenames = yaml.safe_load(f)
+        return filenames
+
+    def path_for_file(self, filename):
+        """
+        Get the path for a file inside the collection.
+        Does not check if the file exists or anything like
+        that.
+        """
+        return str(self.file / filename)
+
+
+class PNGFile(DataFile):
+    suffix = "png"
+
+    @classmethod
+    def open(self, path, mode, **kwargs):
+        import matplotlib
+        import matplotlib.pyplot as plt
+
+        if mode != "w":
+            raise ValueError("Reading existing PNG files is not supported")
+        return plt.figure(**kwargs)
+
+    def close(self):
+        import matplotlib.pyplot as plt
+
+        self.file.savefig(self.path, metadata=self.provenance)
+        plt.close(self.file)
+
+    def write_provenance(self):
+        # provenance is written on closing the file
+        pass
+
+    def read_provenance(self):
+        raise ValueError("Reading existing PNG files is not supported")
+
+
+class PickleFile(DataFile):
+    suffix = "pkl"
+
+    @classmethod
+    def open(self, path, mode, **kwargs):
+        return open(path, mode + "b")
+
+    def write_provenance(self):
+        self.write(self.provenance)
+
+    def read_provenance(self):
+        return self.read()
+
+    def write(self, obj):
+        if self.mode != "w":
+            raise UnsupportedOperation(f"Cannot write to pickle file opened in read-only ({self.mode})")
+        pickle.dump(obj, self.file)
+
+    def read(self):
+        if self.mode != "r":
+            raise UnsupportedOperation(f"Cannot read from pickle file opened in write-only ({self.mode})")
+        return pickle.load(self.file)
+
+
+class ParquetFile(DataFile):
+    suffix = "pq"
+
+    def open(self, path, mode):
+        import pyarrow.parquet
+
+        if mode != "r":
+            raise NotImplementedError("Not implemented writing to Parquet")
+        return pyarrow.parquet.ParquetFile(path)
+
+    def close(self):
+        pass

{ceci-2.3.1 → ceci-2.4}/ceci/pipeline/graph.py

@@ -197,3 +197,44 @@ def trim_pipeline_graph(graph, from_=None, to_=None):
                 converted_inputs[node_name] = file_name
 
     return trimmed_graph, converted_inputs
+
+def combine_output_nodes(graph):
+    """
+    Combine output nodes that share the same parent and have no
+    children into a single node.
+
+    This is useful for plotting flow charts.
+
+    Parameters
+    ----------
+    graph : networkx.DiGraph
+        The pipeline graph to modify
+    """
+    groups = collections.defaultdict(list)
+    # find groups of nodes that have no outgoing edges and all
+    # share the same incoming edge.
+    for node in graph.nodes_iter():
+        node_data = graph.get_node(node).attr
+        if node_data["type"] != "output" or len(graph.successors(node)) != 0:
+            continue
+        predecessors = list(graph.predecessors(node))
+
+        # This shouldn't happen in the default case,
+        # but might if things change later, so let's deal with it
+        # just in case.
+        if len(predecessors) != 1:
+            continue
+
+        parent = predecessors[0]
+        groups[parent].append(node)
+
+    for parent, group in groups.items():
+        if len(group) < 2:
+            continue
+        # make a new node combining all these nodes
+        new_node_name = "\n".join(group)
+        graph.add_node(new_node_name, type="output")
+        # connect the parent to the new node
+        graph.add_edge(parent, new_node_name)
+        # remove the old nodes
+        graph.remove_nodes_from(group)

{ceci-2.3.1 → ceci-2.4}/ceci/pipeline/pipeline.py

@@ -9,7 +9,7 @@ from abc import abstractmethod
 from ..stage import PipelineStage
 from ..sites import load, set_default_site, get_default_site
 from ..utils import extra_paths
-from .graph import build_graph, get_static_ordering, trim_pipeline_graph
+from .graph import build_graph, get_static_ordering, trim_pipeline_graph, combine_output_nodes
 from .file_manager import FileManager
 from .sec import StageExecutionConfig
 from .templates import read_and_apply_template
@@ -898,6 +898,11 @@ class Pipeline:
         # generate a graphviz object from the networkx graph
         graph = networkx.nx_agraph.to_agraph(self.graph)
 
+        # Some stages have long lists of output nodes like plots that
+        # are final outputs not used in the rest of the pipeline. We
+        # can combine these to make the graph more readable.
+        combine_output_nodes(graph)
+
         # set the colours and styles for the boxes
         for node in graph.nodes_iter():
             node_type = node.attr['type']

{ceci-2.3.1 → ceci-2.4}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.3.1
+Version: 2.4
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.3.1 → ceci-2.4}/ceci.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ ceci/__main__.py
 ceci/_version.py
 ceci/config.py
 ceci/errors.py
+ceci/file_types.py
 ceci/main.py
 ceci/minirunner.py
 ceci/monitor.py