ceci 2.3__tar.gz → 2.4__tar.gz

{ceci-2.3 → ceci-2.4}/.coveragerc

@@ -2,7 +2,7 @@
 [run]
 omit=
   *ceci/sites/ccin2p3.py
-  *ceci/sites/cori.py
+  *ceci/sites/nersc.py
 
 
 [html]

ceci-2.4/.readthedocs.yml

@@ -0,0 +1,14 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+  - requirements: docs/requirements.txt
+

{ceci-2.3 → ceci-2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.3
+Version: 2.4
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.3 → ceci-2.4}/ceci/__init__.py

@@ -9,6 +9,7 @@ from .pipeline import (
     FlowChartPipeline,
 )
 from .main import run_pipeline
+from . import file_types
 import importlib.metadata
 
 try:

{ceci-2.3 → ceci-2.4}/ceci/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.3'
-__version_tuple__ = version_tuple = (2, 3)
+__version__ = version = '2.4'
+__version_tuple__ = version_tuple = (2, 4)
 
-__commit_id__ = commit_id = 'ged188fcdb'
+__commit_id__ = commit_id = 'g727f0fadf'

ceci-2.4/ceci/file_types.py

@@ -0,0 +1,496 @@
+import uuid
+import datetime
+import socket
+import getpass
+import warnings
+import pathlib
+import yaml
+import shutil
+import pickle
+from io import UnsupportedOperation
+
+
+class FileValidationError(Exception):
+    pass
+
+
+class DataFile:
+    """
+    A class representing a DataFile to be made by pipeline stages
+    and passed on to subsequent ones.
+
+    DataFile itself should not be instantiated - instead subclasses
+    should be defined for different file types.
+
+    These subclasses are used in the definition of pipeline stages
+    to indicate what kind of file is expected.  The "suffix" attribute,
+    which must be defined on subclasses, indicates the file suffix.
+
+    The open method, which can optionally be overridden, is used by the
+    machinery of the PipelineStage class to open an input our output
+    named by a tag.
+
+    """
+
+    supports_parallel_write = False
+    suffix = None
+
+    def __init__(self, path, mode, extra_provenance=None, validate=True, **kwargs):
+        self.path = path
+        self.mode = mode
+
+        if mode not in ["r", "w"]:
+            raise ValueError(f"File 'mode' argument must be 'r' or 'w' not '{mode}'")
+
+        self.file = self.open(path, mode, **kwargs)
+
+        if validate and mode == "r":
+            self.validate()
+
+        if mode == "w":
+            self.provenance = self.generate_provenance(extra_provenance)
+            self.write_provenance()
+        else:
+            self.provenance = self.read_provenance()
+
+    @staticmethod
+    def generate_provenance(extra_provenance=None):
+        """
+        Generate provenance information - a dictionary
+        of useful information about the origina
+        """
+        UUID = uuid.uuid4().hex
+        creation = datetime.datetime.now().isoformat()
+        domain = socket.getfqdn()
+        username = getpass.getuser()
+
+        # Add other provenance and related
+        provenance = {
+            "uuid": UUID,
+            "creation": creation,
+            "domain": domain,
+            "username": username,
+        }
+
+        if extra_provenance:
+            provenance.update(extra_provenance)
+        return provenance
+
+    def write_provenance(self):
+        """
+        Concrete subclasses (for which it is possible) should override
+        this method to save the dictionary self.provenance to the file.
+        """
+        pass
+
+    def add_provenance(self, key, value):
+        """
+        Concrete subclasses (for which it is possible) should override
+        this method to save the a new string key/value pair to file
+        """
+        pass
+
+    def read_provenance(self):
+        """
+        Concrete subclasses for which it is possible should override
+        this method and read the provenance information from the file.
+
+        Other classes will return this dictionary of UNKNOWNs
+        """
+        provenance = {
+            "uuid": "UNKNOWN",
+            "creation": "UNKNOWN",
+            "domain": "UNKNOWN",
+            "username": "UNKNOWN",
+        }
+
+        return provenance
+
+    def validate(self):
+        """
+        Concrete subclasses should override this method
+        to check that all expected columns are present.
+        """
+        pass
+
+    @classmethod
+    def open(cls, path, mode):
+        """
+        Open a data file.  The base implementation of this function just
+        opens and returns a standard python file object.
+
+        Subclasses can override to either open files using different openers
+        (like fitsio.FITS), or, for more specific data types, return an
+        instance of the class itself to use as an intermediary for the file.
+
+        """
+        return open(path, mode)
+
+    def close(self):
+        self.file.close()
+
+    @classmethod
+    def make_name(cls, tag):
+        if cls.suffix:
+            return f"{tag}.{cls.suffix}"
+        else:
+            return tag
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+
+class HDFFile(DataFile):
+    supports_parallel_write = True
+    """
+    A data file in the HDF5 format.
+    Using these files requires the h5py package, which in turn
+    requires an HDF5 library installation.
+
+    """
+    suffix = "hdf5"
+    required_datasets = []
+
+    @classmethod
+    def open(cls, path, mode, **kwargs):
+        # Suppress a warning that h5py always displays
+        # on import
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            import h5py
+        # Return an open h5py File
+        return h5py.File(path, mode, **kwargs)
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to an HDF5 file opened in read-only mode ({self.mode}")
+
+        # This method *must* be called by all the processes in a parallel
+        # run.
+        self._provenance_group = self.file.create_group("provenance")
+
+        # Call the sub-method to do each item
+        for key, value in self.provenance.items():
+            self._provenance_group.attrs[key] = value
+
+    def read_provenance(self):
+        try:
+            group = self.file["provenance"]
+            attrs = group.attrs
+        except KeyError:
+            group = None
+            attrs = {}
+
+        provenance = {
+            "uuid": attrs.get("uuid", "UNKNOWN"),
+            "creation": attrs.get("creation", "UNKNOWN"),
+            "domain": attrs.get("domain", "UNKNOWN"),
+            "username": attrs.get("username", "UNKNOWN"),
+        }
+
+        self._provenance_group = group
+
+        return provenance
+
+    def validate(self):
+        missing = [name for name in self.required_datasets if name not in self.file]
+        if missing:
+            text = "\n".join(missing)
+            raise FileValidationError(f"These data sets are missing from HDF file {self.path}:\n{text}")
+
+    def close(self):
+        self.file.close()
+
+
+class FitsFile(DataFile):
+    """
+    A data file in the FITS format.
+    Using these files requires the fitsio package.
+    """
+
+    suffix = "fits"
+    required_columns = []
+
+    @classmethod
+    def open(cls, path, mode, **kwargs):
+        import fitsio
+
+        # Fitsio doesn't have pure 'w' modes, just 'rw'.
+        # Maybe we should check if the file already exists here?
+        if mode == "w":
+            mode = "rw"
+        return fitsio.FITS(path, mode=mode, **kwargs)
+
+    def missing_columns(self, columns, hdu=1):
+        """
+        Check that all supplied columns exist
+        and are in the chosen HDU
+        """
+        ext = self.file[hdu]
+        found_cols = ext.get_colnames()
+        missing_columns = [col for col in columns if col not in found_cols]
+        return missing_columns
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        # Call the sub-method to do each item
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to a FITS file opened in read-only mode ({self.mode}")
+
+        for key, value in self.provenance.items():
+            if isinstance(value, str) and "\n" in value:
+                values = value.split("\n")
+                for i, v in enumerate(values):
+                    self.file[0].write_key(key + f"_{i}", v)
+            else:
+                self.file[0].write_key(key, value)
+
+    def read_provenance(self):
+        header = self.file[0].read_header()
+        provenance = {
+            "uuid": header.get("uuid", "UNKNOWN"),
+            "creation": header.get("creation", "UNKNOWN"),
+            "domain": header.get("domain", "UNKNOWN"),
+            "username": header.get("username", "UNKNOWN"),
+        }
+        return provenance
+
+    def validate(self):
+        """Check that the catalog has all the required columns and complain otherwise"""
+        # Find any columns that do not exist in the file
+        missing = self.missing_columns(self.required_columns)
+
+        # If there are any, raise an exception that lists them explicitly
+        if missing:
+            text = "\n".join(missing)
+            raise FileValidationError(f"These columns are missing from FITS file {self.path}:\n{text}")
+
+    def close(self):
+        self.file.close()
+
+
+class TextFile(DataFile):
+    """
+    A data file in plain text format.
+    """
+
+    suffix = "txt"
+
+
+class YamlFile(DataFile):
+    """
+    A data file in yaml format.
+    The top-level object in TXPipe YAML
+    files should always be a dictionary.
+    """
+
+    suffix = "yml"
+
+    def __init__(self, path, mode, extra_provenance=None, validate=True, load_mode="full"):
+        self.path = path
+        self.mode = mode
+        self.file = self.open(path, mode)
+
+        if mode == "r":
+            if load_mode == "safe":
+                self.content = yaml.safe_load(self.file)
+            elif load_mode == "full":
+                self.content = yaml.full_load(self.file)
+            elif load_mode == "unsafe":
+                self.content = yaml.unsafe_load(self.file)
+            else:
+                raise ValueError(f"Unknown value {yaml_load} of load_mode. Should be 'safe', 'full', or 'unsafe'")
+            # get provenance
+            self.provenance = self.read_provenance()
+
+        else:
+            self.provenance = self.generate_provenance(extra_provenance)
+            self.write_provenance()
+
+    def read(self, key):
+        return self.content[key]
+
+    def write(self, d):
+        if not isinstance(d, dict):
+            raise ValueError("Only dicts should be passed to YamlFile.write")
+        yaml.dump(d, self.file)
+
+    def write_provenance(self):
+        d = {"provenance": self.provenance}
+        self.write(d)
+
+    def read_provenance(self):
+        prov = self.content.pop("provenance", {})
+        req_provenance = {
+            "uuid": prov.get("uuid", "UNKNOWN"),
+            "creation": prov.get("creation", "UNKNOWN"),
+            "domain": prov.get("domain", "UNKNOWN"),
+            "username": prov.get("username", "UNKNOWN"),
+        }
+        prov.update(req_provenance)
+        return prov
+
+
+class Directory(DataFile):
+    suffix = ""
+
+    @classmethod
+    def open(self, path, mode):
+        p = pathlib.Path(path)
+
+        if mode == "w":
+            if p.exists():
+                shutil.rmtree(p)
+            p.mkdir(parents=True)
+        else:
+            if not p.is_dir():
+                raise ValueError(f"Directory input {path} does not exist")
+        return p
+
+    def close(self):
+        pass
+
+    def write_provenance(self):
+        """
+        Write provenance information to a new group,
+        called 'provenance'
+        """
+        # This method *must* be called by all the processes in a parallel
+        # run.
+        if self.mode == "r":
+            raise UnsupportedOperation(f"Cannot write provenance to a directory opened in read-only mode ({self.mode})")
+
+        self._provenance_file = open(self.file / "provenance.yml", "w")
+
+        # Call the sub-method to do each item
+        yaml.dump(self.provenance, self._provenance_file)
+        self._provenance_file.flush()
+
+    def read_provenance(self):
+        try:
+            f = open(self.file / "provenance.yml")
+            attrs = yaml.load(f)
+
+        except KeyError:
+            f = None
+            attrs = {}
+
+        self._provenance_file = f
+
+        provenance = {
+            "uuid": attrs.get("uuid", "UNKNOWN"),
+            "creation": attrs.get("creation", "UNKNOWN"),
+            "domain": attrs.get("domain", "UNKNOWN"),
+            "username": attrs.get("username", "UNKNOWN"),
+        }
+
+        return provenance
+
+
+class FileCollection(Directory):
+    """
+    Represents a grouped bundle of files, for cases where you don't
+    know the exact list in advance.
+    """
+
+    suffix = ""
+
+    def write_listing(self, filenames):
+        """
+        Write a listing file in the directory recording
+        (presumably) the filenames put in it.
+        """
+        fn = self.path_for_file("txpipe_listing.txt")
+        with open(fn, "w") as f:
+            yaml.dump(filenames, f)
+
+    def read_listing(self):
+        """
+        Read a listing file from the directory.
+        """
+        fn = self.path_for_file("txpipe_listing.txt")
+        with open(fn, "w") as f:
+            filenames = yaml.safe_load(f)
+        return filenames
+
+    def path_for_file(self, filename):
+        """
+        Get the path for a file inside the collection.
+        Does not check if the file exists or anything like
+        that.
+        """
+        return str(self.file / filename)
+
+
+class PNGFile(DataFile):
+    suffix = "png"
+
+    @classmethod
+    def open(self, path, mode, **kwargs):
+        import matplotlib
+        import matplotlib.pyplot as plt
+
+        if mode != "w":
+            raise ValueError("Reading existing PNG files is not supported")
+        return plt.figure(**kwargs)
+
+    def close(self):
+        import matplotlib.pyplot as plt
+
+        self.file.savefig(self.path, metadata=self.provenance)
+        plt.close(self.file)
+
+    def write_provenance(self):
+        # provenance is written on closing the file
+        pass
+
+    def read_provenance(self):
+        raise ValueError("Reading existing PNG files is not supported")
+
+
+class PickleFile(DataFile):
+    suffix = "pkl"
+
+    @classmethod
+    def open(self, path, mode, **kwargs):
+        return open(path, mode + "b")
+
+    def write_provenance(self):
+        self.write(self.provenance)
+
+    def read_provenance(self):
+        return self.read()
+
+    def write(self, obj):
+        if self.mode != "w":
+            raise UnsupportedOperation(f"Cannot write to pickle file opened in read-only ({self.mode})")
+        pickle.dump(obj, self.file)
+
+    def read(self):
+        if self.mode != "r":
+            raise UnsupportedOperation(f"Cannot read from pickle file opened in write-only ({self.mode})")
+        return pickle.load(self.file)
+
+
+class ParquetFile(DataFile):
+    suffix = "pq"
+
+    def open(self, path, mode):
+        import pyarrow.parquet
+
+        if mode != "r":
+            raise NotImplementedError("Not implemented writing to Parquet")
+        return pyarrow.parquet.ParquetFile(path)
+
+    def close(self):
+        pass

{ceci-2.3 → ceci-2.4}/ceci/pipeline/graph.py

@@ -71,7 +71,7 @@ def build_graph(stage_names, stage_classes, stage_aliases, overall_inputs):
         for tag, stages in missing.items():
             stages = ", ".join(stages)
             msg.append(f"- {tag} - needed by stages {stages}")
-        msg = "\n.join(msg)"
+        msg = "\n".join(msg)
         raise ValueError(f"The following files are needed by the pipeline but not generated by any stage:\n{msg}")
 
     # Now check for cycles in the graph
@@ -197,3 +197,44 @@ def trim_pipeline_graph(graph, from_=None, to_=None):
                 converted_inputs[node_name] = file_name
 
     return trimmed_graph, converted_inputs
+
+def combine_output_nodes(graph):
+    """
+    Combine output nodes that share the same parent and have no
+    children into a single node.
+
+    This is useful for plotting flow charts.
+
+    Parameters
+    ----------
+    graph : networkx.DiGraph
+        The pipeline graph to modify
+    """
+    groups = collections.defaultdict(list)
+    # find groups of nodes that have no outgoing edges and all
+    # share the same incoming edge.
+    for node in graph.nodes_iter():
+        node_data = graph.get_node(node).attr
+        if node_data["type"] != "output" or len(graph.successors(node)) != 0:
+            continue
+        predecessors = list(graph.predecessors(node))
+
+        # This shouldn't happen in the default case,
+        # but might if things change later, so let's deal with it
+        # just in case.
+        if len(predecessors) != 1:
+            continue
+
+        parent = predecessors[0]
+        groups[parent].append(node)
+
+    for parent, group in groups.items():
+        if len(group) < 2:
+            continue
+        # make a new node combining all these nodes
+        new_node_name = "\n".join(group)
+        graph.add_node(new_node_name, type="output")
+        # connect the parent to the new node
+        graph.add_edge(parent, new_node_name)
+        # remove the old nodes
+        graph.remove_nodes_from(group)

{ceci-2.3 → ceci-2.4}/ceci/pipeline/mini.py

@@ -6,8 +6,8 @@ class MiniPipeline(Pipeline):
     """A pipeline subclass that uses Minirunner, a sub-module
     of ceci, to run.
 
-    Minirununer is a small tool I wrote suitable for interactive
-    jobs on cori.  It launches jobs locally (not through a batch system),
+    Minirunner is a small tool I wrote suitable for interactive
+    jobs on NERSC.  It launches jobs locally (not through a batch system),
     which parsl can also do, but it has a simple and clearer (to me at least)
     understanding of available nodes and cores.
 

{ceci-2.3 → ceci-2.4}/ceci/pipeline/pipeline.py

@@ -9,7 +9,7 @@ from abc import abstractmethod
 from ..stage import PipelineStage
 from ..sites import load, set_default_site, get_default_site
 from ..utils import extra_paths
-from .graph import build_graph, get_static_ordering, trim_pipeline_graph
+from .graph import build_graph, get_static_ordering, trim_pipeline_graph, combine_output_nodes
 from .file_manager import FileManager
 from .sec import StageExecutionConfig
 from .templates import read_and_apply_template
@@ -898,6 +898,11 @@ class Pipeline:
         # generate a graphviz object from the networkx graph
         graph = networkx.nx_agraph.to_agraph(self.graph)
 
+        # Some stages have long lists of output nodes like plots that
+        # are final outputs not used in the rest of the pipeline. We
+        # can combine these to make the graph more readable.
+        combine_output_nodes(graph)
+
         # set the colours and styles for the boxes
         for node in graph.nodes_iter():
             node_type = node.attr['type']

{ceci-2.3 → ceci-2.4}/ceci/sites/nersc.py

@@ -149,7 +149,7 @@ class NerscBatchSite(NerscSite):
         walltime = self.config.get("walltime", "00:30:00")
         setup_script = self.config.get(
             "setup",
-            "/global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori",
+            "/dev/null",
         )
 
         provider = SlurmProvider(

{ceci-2.3 → ceci-2.4}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.3
+Version: 2.4
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.3 → ceci-2.4}/ceci.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 .coveragerc
 .gitignore
 .pylintrc
+.readthedocs.yml
 LICENSE
 README.md
 ceci.png
@@ -14,6 +15,7 @@ ceci/__main__.py
 ceci/_version.py
 ceci/config.py
 ceci/errors.py
+ceci/file_types.py
 ceci/main.py
 ceci/minirunner.py
 ceci/monitor.py
@@ -57,6 +59,7 @@ docs/installation.rst
 docs/launchers.rst
 docs/logo.png
 docs/overview.rst
+docs/requirements.txt
 docs/sites.rst
 docs/stages.rst
 docs/tutorial.rst

{ceci-2.3 → ceci-2.4}/docs/conf.py

@@ -20,13 +20,13 @@ sys.path.insert(0, os.path.abspath('..'))
 # -- Project information -----------------------------------------------------
 
 project = 'ceci'
-copyright = '2018, Joe Zuntz'
-author = 'Joe Zuntz'
+copyright = '2018, LSSTDESC'
+author = 'Joe Zuntz, LSSTDESC'
 
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '0.0.1'
+release = '2.3'
 
 
 # -- General configuration ---------------------------------------------------
@@ -41,7 +41,7 @@ release = '0.0.1'
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
-    'sphinx.ext.intersphinx',
+    # 'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
     'sphinx.ext.mathjax',
@@ -169,7 +169,7 @@ texinfo_documents = [
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+# intersphinx_mapping = {'https://docs.python.org/': None}
 
 # -- Options for todo extension ----------------------------------------------
 

{ceci-2.3 → ceci-2.4}/docs/config.rst

@@ -16,7 +16,7 @@ Here is an example, from ``test/test.yml``.  The different pieces are described
       name: mini
       interval: 0.5
   # and three sites:
-  # local, cori, and cori-interactive
+  # local, nersc-batch, and nersc-interactive
   site:
       name: local
       max_threads: 2
@@ -114,7 +114,7 @@ Site
 
 The ``site`` parameter should be a dictionary that configures the machine on which you are running the pipeline.
 
-The ``name`` item in the dictionary sets which site is used.  These options are currently allowed: ``local``, ``cori-batch``, and ``cori-interactive``.
+The ``name`` item in the dictionary sets which site is used.  These options are currently allowed: ``local``, ``nersc-batch``, and ``nersc-interactive``.
 
 See the :ref:`sites` page for information on these sites, and the other options they take.
 
@@ -161,3 +161,25 @@ Any executable specified by ``pre_script`` will be run before the pipeline.  If
 Any executable specified by ``post_script`` will be run after the pipeline, but only if the pipeline completes successfully.  If the post_script returns a non-zero status then it will be returned as the ceci exit code, but no exception will be raised.
 
 Both scripts are called with the same arguments as the original executable was called with.
+
+
+Templates
+---------
+
+You can use `Jinja2 <https://jinja.palletsprojects.com>`_ to allow you to use *templates* in your pipeline YML files.
+
+This allows you to set variables on the ceci command line which are then used to
+modify the pipeline text. This lets you use a single parameter file for a set of
+runs. In TXPipe we use this to have a single pipeline file that can process one of several
+different fields depending on the parameter choice.
+
+See the `Jinja2 <https://jinja.palletsprojects.com>`_ documentation for information on the
+template syntax. Then to set input variables, use the `-t` flag on the ceci command line, in the form:
+
+.. code-block:: bash
+
+  ceci -t variable1=value1 variable2=value2   pipeline.yml
+
+
+There's not much point using template in interactive code - you should probably
+just set up your pipeline files programatically.

{ceci-2.3 → ceci-2.4}/docs/launchers.rst

@@ -14,9 +14,9 @@ Minirunner
 
 The ``mini`` launcher is a minimal in-built launcher with only basic features, but it's useful for small to medium sized jobs.   
 
-Minirunner understands the concept of Nodes versus Cores on supercomputers, and on Cori the numbers are determined from SLURM environment variables.   If running on the login node, one node with four cores is assigned.
+Minirunner understands the concept of Nodes versus Cores on supercomputers, and on NERSC the numbers are determined from SLURM environment variables.   If running on the login node, one node with four cores is assigned.
 
-Minirunner does not launch jobs - if you want to use it in Cori batch mode you should call it from within the job submission script.
+Minirunner does not launch jobs - if you want to use it in NERSC batch mode you should call it from within the job submission script.
 
 Minirunner options
 ^^^^^^^^^^^^^^^^^^

ceci-2.4/docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme

{ceci-2.3 → ceci-2.4}/docs/sites.rst

@@ -5,7 +5,7 @@ Sites
 
 A site is a machine where a pipeline is to be run.  Ceci currently only supports running a pipeline at a single site, not splitting it up between them.
 
-Three sites are currently supported: ``local``, ``cori-batch``, and ``cori-interactive``.
+Three sites are currently supported: ``local``, ``nersc-batch``, and ``nersc-interactive``.
 
 See also the :ref:`launchers` page for how to configure the manager that runs the pipeline.
 
@@ -45,35 +45,31 @@ The local site is a general one and represents running in a straightforward loca
 ``max_threads`` is optional and controls the maximum number of stages run at the same time.  Its default depends on the launcher used.
 
 
-Cori Interactive
-----------------
+NERSC Interactive
+------------------
 
-The ``cori-interactive`` site is used to run jobs interactively on NERSC compute nodes.  You should first use the ``salloc`` command to get an interactive allocation, and then within that run ``ceci``.
+The ``nersc-interactive`` site is used to run jobs interactively on NERSC compute nodes.  You should first use the ``salloc`` command to get an interactive allocation, and then within that run ``ceci``.
 
-There are no additional options for the ``cori-interactive`` site: the number of parallel stages is given by the number of nodes that you ask for in ``salloc``.
+There are no additional options for the ``nersc-interactive`` site: the number of parallel stages is given by the number of nodes that you ask for in ``salloc``.
 
 
 
-Cori Batch
-----------
+NERSC Batch
+------------
 
-The site ``cori-batch`` runs on the Cori supercomputer at NERSC, and submits jobs to the SLURM batch system.  In this mode, you should call ceci from the login node and stay logged in while the jobs run.
+The site ``nersc-batch`` runs on the Perlmutter supercomputer at NERSC, and submits jobs to the SLURM batch system.  In this mode, you should call ceci from the login node and stay logged in while the jobs run.
 
-These options can be used for the ``cori-batch`` site:
+These options can be used for the ``nersc-batch`` site:
 
 .. code-block:: yaml
 
     launcher:
-        name: cori
-        cpu_type: haswell   # optional
+        name: nersc-interactive
         queue: debug        # optional
         max_jobs: 2         # optional
         account: m1727      # optional
         walltime: 00:30:00  # optional
-        setup:  /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori
-        # ^^ optional
 
-``cpu_type`` is optional and controls which partition of cori is used for jobs, and should be `haswell` or `KNL`.
 
 ``queue`` is optional and controls which SLURM queue jobs are launcher on. It can be ``debug``, ``regular``, or ``premium``. See `the nersc documentation <http://docs.nersc.gov/>`_
  for a description of each.

{ceci-2.3 → ceci-2.4}/tests/template.yml

@@ -42,7 +42,6 @@ site:
 #     max_jobs: 2
 #     account: m1727
 #     walltime: "00:30:00"
-#     setup: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori
 
 
 

{ceci-2.3 → ceci-2.4}/tests/test.yml

@@ -36,7 +36,6 @@ site:
 #     max_jobs: 2
 #     account: m1727
 #     walltime: "00:30:00"
-#     setup: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori
 
 
 

{ceci-2.3 → ceci-2.4}/tests/test_dataregistry.yml

@@ -36,7 +36,6 @@ site:
 #     max_jobs: 2
 #     account: m1727
 #     walltime: "00:30:00"
-#     setup: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori
 
 
 

{ceci-2.3 → ceci-2.4}/tests/test_namespace.yml

@@ -36,7 +36,6 @@ site:
 #     max_jobs: 2
 #     account: m1727
 #     walltime: "00:30:00"
-#     setup: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/setup-cori