ceci 2.4__tar.gz → 2.5__tar.gz

{ceci-2.4 → ceci-2.5}/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
+        python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", "3.14"]
   
     steps:
     - name: Checkout repository

{ceci-2.4 → ceci-2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.4
+Version: 2.5
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

ceci-2.5/ceci/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '2.5'
+__version_tuple__ = version_tuple = (2, 5)
+
+__commit_id__ = commit_id = 'gfccc21033'

{ceci-2.4 → ceci-2.5}/ceci/file_types.py

@@ -39,6 +39,11 @@ class DataFile:
         self.path = path
         self.mode = mode
 
+        # store whether we are in parallel mode
+        # for later use.
+        self.parallel = kwargs.get("parallel", False)
+        self.comm = kwargs.get("comm", None)
+
         if mode not in ["r", "w"]:
             raise ValueError(f"File 'mode' argument must be 'r' or 'w' not '{mode}'")
 
@@ -343,15 +348,24 @@ class YamlFile(DataFile):
 
 class Directory(DataFile):
     suffix = ""
+    supports_parallel_write = True
 
     @classmethod
-    def open(self, path, mode):
+    def open(self, path, mode, comm=None, parallel=False):
         p = pathlib.Path(path)
 
         if mode == "w":
-            if p.exists():
-                shutil.rmtree(p)
-            p.mkdir(parents=True)
+            # Make it so that only one process
+            # tries to create the directory if running
+            # in parallel
+            if (not parallel) or (parallel and comm.rank == 0):
+                if p.exists():
+                    shutil.rmtree(p)
+                p.mkdir(parents=True)
+            # Then once that one process as done it all the others
+            # should wait to avoid race conditions.
+            if parallel:
+                comm.Barrier()
         else:
             if not p.is_dir():
                 raise ValueError(f"Directory input {path} does not exist")
@@ -365,8 +379,13 @@ class Directory(DataFile):
         Write provenance information to a new group,
         called 'provenance'
         """
-        # This method *must* be called by all the processes in a parallel
-        # run.
+        # Directories can be opened in parallel, but only
+        # the rank 0 process should write the provenance.
+        # This is unlike HDF5 where all the processes have to
+        # cooperate to write it.
+        if (self.comm is not None) and (self.comm.rank != 0):
+            return
+
         if self.mode == "r":
             raise UnsupportedOperation(f"Cannot write provenance to a directory opened in read-only mode ({self.mode})")
 

{ceci-2.4 → ceci-2.5}/ceci/stage.py

@@ -16,6 +16,7 @@ from . import errors
 from .monitor import MemoryMonitor
 from .config import StageParameter, StageConfig, cast_to_streamable
 from .utils import activate_tracing
+from . import file_types
 
 SERIAL = "serial"
 MPI_PARALLEL = "mpi"
@@ -1131,13 +1132,19 @@ I currently know about these stages:
         path = self.get_output(tag, final_name=final_name)
         output_class = self.get_output_type(tag)
 
-        # HDF files can be opened for parallel writing
+        # HDF files and directory outputs can be opened for parallel writing
         # under MPI.  This checks if:
         # - we have been told to open in parallel
         # - we are actually running under MPI
         # and adds the flags required if all these are true
         run_parallel = kwargs.pop("parallel", False) and self.is_mpi()
-        if run_parallel:
+        if run_parallel and issubclass(output_class, file_types.Directory):
+            kwargs["parallel"] = True
+            kwargs["comm"] = self.comm
+
+        # otherwise must be HDF5. Ideally we would check more carefully
+        # here, but I don't want to mess up any RAIL HDF stuff by accident.
+        elif run_parallel:
             kwargs["driver"] = "mpio"
             kwargs["comm"] = self.comm
 

{ceci-2.4 → ceci-2.5}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ceci
-Version: 2.4
+Version: 2.5
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.4 → ceci-2.5}/ceci.egg-info/SOURCES.txt

@@ -80,6 +80,7 @@ tests/test_flow_chart.py
 tests/test_graph.py
 tests/test_helpers.py
 tests/test_interactive.py
+tests/test_io.py
 tests/test_main.py
 tests/test_minirunner.py
 tests/test_monitor.py

ceci-2.5/tests/test_io.py

@@ -0,0 +1,24 @@
+import ceci.file_types as file_types
+import tempfile
+import os
+import mockmpi
+
+
+def _test_directory_parallel(comm, tmpdir):
+        print(comm.rank, tmpdir)
+        d = file_types.Directory(tmpdir + "/test_dir", "w", parallel=True, comm=comm)
+        comm.Barrier()
+        assert d.path == tmpdir + "/test_dir"
+        assert os.path.exists(tmpdir + "/test_dir/provenance.yml")
+
+
+def test_directory():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        d = file_types.Directory(tmpdir + "/test_dir", "w", parallel=False)
+        assert d.path == tmpdir + "/test_dir"
+        assert os.path.exists(tmpdir + "/test_dir/provenance.yml")
+
+
+def test_directory_parallel():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        mockmpi.mock_mpiexec(2, _test_directory_parallel, tmpdir)

ceci-2.4/ceci/_version.py

@@ -1,34 +0,0 @@
-# file generated by setuptools-scm
-# don't change, don't track in version control
-
-__all__ = [
-    "__version__",
-    "__version_tuple__",
-    "version",
-    "version_tuple",
-    "__commit_id__",
-    "commit_id",
-]
-
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-    COMMIT_ID = Union[str, None]
-else:
-    VERSION_TUPLE = object
-    COMMIT_ID = object
-
-version: str
-__version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-commit_id: COMMIT_ID
-__commit_id__: COMMIT_ID
-
-__version__ = version = '2.4'
-__version_tuple__ = version_tuple = (2, 4)
-
-__commit_id__ = commit_id = 'g727f0fadf'