ceci 2.7__tar.gz → 2.7.2__tar.gz

{ceci-2.7 → ceci-2.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.7
+Version: 2.7.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.7 → ceci-2.7.2}/ceci/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.7'
-__version_tuple__ = version_tuple = (2, 7)
+__version__ = version = '2.7.2'
+__version_tuple__ = version_tuple = (2, 7, 2)
 
-__commit_id__ = commit_id = 'g6fabd7fa4'
+__commit_id__ = commit_id = 'g719607c5f'

{ceci-2.7 → ceci-2.7.2}/ceci/pipeline/pipeline.py

@@ -46,7 +46,7 @@ def prepare_for_pipeline(pipe_config):
 
     # Get information (maybe the default) on the launcher we may be using
     launcher_config = pipe_config.setdefault("launcher", {"name": "mini"})
-    site_config = pipe_config.get("site", {"name": "local"})
+    site_config = pipe_config.get("site", {"name": "auto"})
 
     # Pass the paths along to the site
     site_config["python_paths"] = paths
@@ -495,8 +495,8 @@ class Pipeline:
 
         modules = pipe_config["modules"].split()
         launcher_config = pipe_config.setdefault("launcher", {"name": "mini"})
-        site_config = dict(name="local")
-        site_config.update(**pipe_config.get("site"))
+        site_config = dict(name="auto")
+        site_config.update(**(pipe_config.get("site") or {}))
         # Pass the paths along to the site
         site_config["python_paths"] = paths
         load(launcher_config, [site_config])
@@ -652,6 +652,9 @@ class Pipeline:
                 # Find the inputs for this stage and set up the arguments
                 # to the stage init method
                 args = stage_config.copy()
+                # Make sure the instance name is the pipeline name, which
+                # may differ from the class name if 'classname' was given
+                args.setdefault("name", stage_name)
                 for tag in stage_class.input_tags():
                     aliased_tag = stage_aliases.get(tag, tag)
                     args[aliased_tag] = self.graph.nodes[aliased_tag]["path"]

{ceci-2.7 → ceci-2.7.2}/ceci/sites/__init__.py

@@ -43,6 +43,13 @@ def get_default_site():
 reset_default_site()
 
 
+def detect_site_name():
+    """Choose "nersc-interactive" if NERSC_HOST is set, otherwise "local"."""
+    if "NERSC_HOST" in os.environ:
+        return "nersc-interactive"
+    return "local"
+
+
 def load(launcher_config, site_configs):
     """Configure a launcher and the sites it will execute on.
 
@@ -70,7 +77,10 @@ def load(launcher_config, site_configs):
 
     # Create an object for each site.
     for site_config in site_configs:
-        site_name = site_config["name"]
+        site_name = site_config.get("name", "auto")
+        if site_name == "auto":
+            site_name = detect_site_name()
+        site_config["name"] = site_name
         # Also tell the sites whether this is a dry-run.
         # for example, the nersc site checks you're not
         # trying to run srun on a login node, but we skip

{ceci-2.7 → ceci-2.7.2}/ceci/sites/nersc.py

@@ -108,7 +108,7 @@ class NerscSite(Site):
             slurm_cpus_on_node = int(os.environ.get("SLURM_CPUS_ON_NODE", 32))
 
             # collect list.
-            nodes = [Node(name, cpus_per_node) for name in node_names]
+            nodes = [Node(name, slurm_cpus_on_node) for name in node_names]
         else:
             # running on login node
             # use at most 4 procs to avoid annoying people
@@ -147,7 +147,7 @@ class NerscBatchSite(NerscSite):
             max_blocks=max_slurm_jobs,  # one slurm job to start with
             scheduler_options=f"#SBATCH --constraint={cpu_type}\n"
             f"#SBATCH --account={account}\n"
-            f"#SBATCH --walltime={walltime}\n",
+            f"#SBATCH --time={walltime}\n",
             nodes_per_block=1,
             init_blocks=1,
             worker_init=f"source {setup_script}",

{ceci-2.7 → ceci-2.7.2}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.7
+Version: 2.7.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.7 → ceci-2.7.2}/ceci.egg-info/SOURCES.txt

@@ -73,6 +73,7 @@ tests/north.yml
 tests/template.yml
 tests/test.hdf5
 tests/test.yml
+tests/test_classname.py
 tests/test_cmd.py
 tests/test_dask.py
 tests/test_dataregistry.yml

{ceci-2.7 → ceci-2.7.2}/ceci.egg-info/scm_file_list.json

@@ -68,6 +68,7 @@
     "tests/template.yml",
     "tests/test.hdf5",
     "tests/test.yml",
+    "tests/test_classname.py",
     "tests/test_cmd.py",
     "tests/test_dask.py",
     "tests/test_dataregistry.yml",

ceci-2.7.2/ceci.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "2.7.2",
+  "distance": 0,
+  "node": "g719607c5fd9b6423c9657a84e4b91bbeaa9f767c",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-10-28"
+}

{ceci-2.7 → ceci-2.7.2}/docs/config.rst

@@ -114,7 +114,7 @@ Site
 
 The ``site`` parameter should be a dictionary that configures the machine on which you are running the pipeline.
 
-The ``name`` item in the dictionary sets which site is used.  These options are currently allowed: ``local``, ``nersc-batch``, and ``nersc-interactive``.
+The ``name`` item in the dictionary sets which site is used.  These options are currently allowed: ``auto``, ``local``, ``nersc-batch``, and ``nersc-interactive``. The default, ``auto``, selects ``nersc-interactive`` if the ``NERSC_HOST`` environment variable is set, and ``local`` otherwise.
 
 See the :ref:`sites` page for information on these sites, and the other options they take.
 

ceci-2.7.2/tests/test_classname.py

@@ -0,0 +1,29 @@
+import ceci_example  # noqa: F401  (registers the example stages)
+import yaml
+from ceci import DryRunPipeline
+from ceci.pipeline.templates import read_and_apply_template
+
+
+def test_stage_name_differs_from_classname():
+    """A stage whose pipeline name differs from its class name should
+    be set up with the pipeline name as its instance name."""
+    config = yaml.safe_load(read_and_apply_template("tests/test.yml"))
+    stages = []
+    for stage in config["stages"]:
+        if stage["name"] == "WLGCSelector":
+            stage = {**stage, "name": "MySelector", "classname": "WLGCSelector"}
+        stages.append(stage)
+
+    pipeline = DryRunPipeline(stages, {"interval": 0.5, "name": "mini"})
+    run_config = {
+        "log_dir": config["log_dir"],
+        "output_dir": config["output_dir"],
+        "resume": False,
+    }
+    pipeline.initialize(config["inputs"], run_config, config["config"])
+
+    assert "MySelector" in pipeline.stage_names
+    assert "WLGCSelector" not in pipeline.stage_names
+    instance_names = [s.instance_name for s in pipeline.stages]
+    assert instance_names == pipeline.stage_names
+    assert pipeline.run() == 0

ceci-2.7.2/tests/test_site.py

@@ -0,0 +1,135 @@
+from ceci.sites import load, get_default_site
+from ceci.pipeline import StageExecutionConfig
+import pytest
+import os
+
+def test_nersc_error():
+    # check that errors when trying to run multi-process
+    # jobs on nersc login nodes are handled correctly.
+    # should fail unless dry-run is set.
+
+    launcher_config = {
+        "name": "mini",
+        "interval": 1.0,
+    }
+    site_config = {
+        "name": "nersc-interactive",
+    }
+
+    stage_config = {
+        "name": "Test",
+        "nprocess": 2,
+    }
+
+    load(launcher_config, [site_config])
+    site = get_default_site()
+    sec = StageExecutionConfig(stage_config)
+
+    # should fail if we don't set dry-run
+    with pytest.raises(ValueError):
+        site.command("xxx", sec)
+
+    # should work if we do set dry-run
+    launcher_config["dry_run"] = True
+    load(launcher_config, [site_config])
+    site = get_default_site()
+    site.command("xxx", sec)
+
+
+def test_in2p3():
+    os.environ["NSLOTS"] = "8"
+
+    launcher_config = {
+        "name": "mini",
+        "interval": 1.0,
+    }
+    site_config = {
+        "name": "cc-parallel",
+    }
+
+    stage_config = {
+        "name": "Test",
+        "nprocess": 2,
+    }
+
+    load(launcher_config, [site_config])
+    site = get_default_site()
+    sec = StageExecutionConfig(stage_config)
+
+    # should work if we do set dry-run
+    launcher_config["dry_run"] = True
+    load(launcher_config, [site_config])
+    site = get_default_site()
+    cmd = site.command("xxx", sec)
+    assert "singularity" not in cmd
+
+    stage_config["image"] = "imaginary_file.sif"
+    sec = StageExecutionConfig(stage_config)
+
+
+    load(launcher_config, [site_config])
+    site = get_default_site()
+    cmd = site.command("xxx", sec)
+    assert "singularity run" in cmd
+
+
+def test_nersc_parse_int_set():
+    from ceci.sites.nersc import parse_int_set
+
+    assert parse_int_set("1,3-5,9") == {1, 3, 4, 5, 9}
+    with pytest.raises(ValueError):
+        parse_int_set("1,abc")
+
+
+def test_nersc_mini_login_node(monkeypatch):
+    from ceci.sites.nersc import NerscSite
+
+    monkeypatch.delenv("SLURM_JOB_ID", raising=False)
+    site = NerscSite({})
+    site.configure_for_mini()
+    (node,) = site.info["nodes"]
+    assert node.cores == 4
+
+
+def test_nersc_mini_slurm_job(monkeypatch):
+    # Simulate being inside a SLURM job, so this can run anywhere (e.g. CI)
+    from ceci.sites.nersc import NerscSite
+
+    monkeypatch.setenv("SLURM_JOB_ID", "12345")
+    monkeypatch.setenv("SLURM_JOB_NODELIST", "nid[1234-1235,1240]")
+    monkeypatch.setenv("SLURM_CPUS_ON_NODE", "64")
+    site = NerscSite({})
+    site.configure_for_mini()
+    nodes = site.info["nodes"]
+    assert len(nodes) == 3
+    assert all(node.cores == 64 for node in nodes)
+
+    # single node, no brackets, and the default core count
+    monkeypatch.setenv("SLURM_JOB_NODELIST", "nid001234")
+    monkeypatch.delenv("SLURM_CPUS_ON_NODE")
+    site = NerscSite({})
+    site.configure_for_mini()
+    (node,) = site.info["nodes"]
+    assert node.id == "nid001234"
+    assert node.cores == 32
+
+
+def test_nersc_command(monkeypatch):
+    from ceci.sites.nersc import NerscSite
+
+    monkeypatch.setenv("SLURM_JOB_ID", "12345")
+    site = NerscSite({"python_paths": ["/a", "/b"]})
+
+    sec = StageExecutionConfig({"name": "Test", "nprocess": 4, "threads_per_process": 2})
+    cmd = site.command("xxx", sec)
+    assert "srun -u -n 4 --cpus-per-task=2" in cmd
+    assert "OMP_NUM_THREADS=2" in cmd
+    assert "PYTHONPATH=/a:/b:$PYTHONPATH" in cmd
+    assert "shifter" not in cmd
+    assert "--mpi" in cmd
+
+    sec = StageExecutionConfig({"name": "Test", "nprocess": 1, "image": "img:latest"})
+    cmd = site.command("xxx", sec)
+    assert "shifter" in cmd
+    assert "--image img:latest" in cmd
+    assert "--mpi" not in cmd

ceci-2.7/ceci.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "2.7",
-  "distance": 0,
-  "node": "g6fabd7fa413a35870896f0eca83d29c3bf24638d",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-10-26"
-}

ceci-2.7/tests/test_site.py

@@ -1,73 +0,0 @@
-from ceci.sites import load, get_default_site
-from ceci.pipeline import StageExecutionConfig
-import pytest
-import os
-
-def test_nersc_error():
-    # check that errors when trying to run multi-process
-    # jobs on nersc login nodes are handled correctly.
-    # should fail unless dry-run is set.
-
-    launcher_config = {
-        "name": "mini",
-        "interval": 1.0,
-    }
-    site_config = {
-        "name": "nersc-interactive",
-    }
-
-    stage_config = {
-        "name": "Test",
-        "nprocess": 2,
-    }
-
-    load(launcher_config, [site_config])
-    site = get_default_site()
-    sec = StageExecutionConfig(stage_config)
-
-    # should fail if we don't set dry-run
-    with pytest.raises(ValueError):
-        site.command("xxx", sec)
-
-    # should work if we do set dry-run
-    launcher_config["dry_run"] = True
-    load(launcher_config, [site_config])
-    site = get_default_site()
-    site.command("xxx", sec)
-
-
-def test_in2p3():
-    os.environ["NSLOTS"] = "8"
-
-    launcher_config = {
-        "name": "mini",
-        "interval": 1.0,
-    }
-    site_config = {
-        "name": "cc-parallel",
-    }
-
-    stage_config = {
-        "name": "Test",
-        "nprocess": 2,
-    }
-
-    load(launcher_config, [site_config])
-    site = get_default_site()
-    sec = StageExecutionConfig(stage_config)
-
-    # should work if we do set dry-run
-    launcher_config["dry_run"] = True
-    load(launcher_config, [site_config])
-    site = get_default_site()
-    cmd = site.command("xxx", sec)
-    assert "singularity" not in cmd
-
-    stage_config["image"] = "imaginary_file.sif"
-    sec = StageExecutionConfig(stage_config)
-
-
-    load(launcher_config, [site_config])
-    site = get_default_site()
-    cmd = site.command("xxx", sec)
-    assert "singularity run" in cmd