ceci 2.1__tar.gz → 2.2__tar.gz

{ceci-2.1 → ceci-2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ceci
-Version: 2.1
+Version: 2.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.1 → ceci-2.2}/ceci/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.1'
-__version_tuple__ = version_tuple = (2, 1)
+__version__ = version = '2.2'
+__version_tuple__ = version_tuple = (2, 2)

{ceci-2.1 → ceci-2.2}/ceci/config.py

@@ -88,6 +88,10 @@ class StageParameter:
             self._required = self._default is None
         self._value = cast_value(self._dtype, self._default)
 
+    def __repr__(self):
+        req_str = 'required' if self.required else 'optional'
+        return f"Parameter({self.msg}, type: {self.dtype}, default: {self.default} [{req_str}])"
+
     @property
     def value(self):
         """Return the value"""
@@ -138,6 +142,27 @@ class StageParameter:
         self._default = default
         self.set_to_default()
 
+    def numpy_style_help_text(self):
+        """Create a docstring followwing numpy style guidelines"""
+        s = ""
+        if self._dtype is None:
+            s += "[type not specified] "
+        else:
+            s += f"[{self._dtype.__name__}] "
+        if self._required:
+            s += "(required)"
+        else:
+            # truncate long dicts
+            if isinstance(self._default, dict) and len(self._default) > 10:
+                s += "(default={...})"
+            # truncate long lists
+            elif isinstance(self._default, list) and len(self._default) > 10:
+                s += "(default=[...])"
+            else:
+                s += f"default={self._default}"
+        s += f"\n    {self._help}"
+        return s
+
 
 class StageConfig(dict):
     """A small class to manage a dictionary of configuration parameters with basic type checking"""
@@ -279,3 +304,19 @@ class StageConfig(dict):
         if isinstance(attr, StageParameter):
             return attr.dtype
         return type(attr)
+
+    def numpy_style_help_text(self):
+        """Create a docstring followwing numpy style guidelines"""
+        if not self:
+            return "<This class has no configuration options>"
+        s = []
+        for key, val in dict.items(self):
+            if isinstance(val, StageConfig):
+                val = val[key]
+            if isinstance(val, StageParameter):
+                s.append(f"{key}: {val.numpy_style_help_text()}")
+            elif isinstance(val, type):
+                s.append(f"{key}: {val.__name__}] (required)")
+            else:
+                s.append(f"{key}: {type(val).__name__}] (default={val})")
+        return '\n\n'.join(s)

{ceci-2.1 → ceci-2.2}/ceci/pipeline.py

@@ -524,18 +524,23 @@ class Pipeline:
         else:
             raise ValueError("Must specify either id or name in registry lookup")
 
+        # Whatever the lookup, we always require a dataset which has not beed deleted or replaced
+        status_filter = registry.Query.gen_filter("dataset.status", "==", 1)
+
         # Main finder method
-        results = registry.Query.find_datasets(["dataset.dataset_id"], [filter])
+        results = registry.Query.find_datasets(["dataset.dataset_id"], [filter, status_filter])
 
         # Check that we find exactly one dataset matching the query
-        results = list(results)
-        if len(results) == 0:
+        if not results:
             raise ValueError(f"Could not find any dataset matching {info} in registry")
-        elif len(results) > 1:
+
+        results = results['dataset.dataset_id']
+
+        if len(results) > 1:
             raise ValueError(f"Found multiple datasets matching {info} in registry")
 
         # Get the absolute path
-        return registry.Query.get_dataset_absolute_path(results[0].dataset_id)
+        return registry.Query.get_dataset_absolute_path(results[0])
 
 
     def process_overall_inputs(self, inputs):

{ceci-2.1 → ceci-2.2}/ceci/stage.py

@@ -153,13 +153,13 @@ class PipelineStage:
         Subclasses must implemented this method.
         """
         raise NotImplementedError("run")
-    
+
     def validate(self):
         """Check that the inputs actually have the data needed for execution,
         This is called before the run method. It is an optional stage, meant
         for checking that the input to the stage is actual in the form and
         shape needed before an expensive run is executed."""
-        pass 
+        pass
 
     def load_configs(self, args):
         """
@@ -356,11 +356,11 @@ class PipelineStage:
         if stage_is_complete and cls.config_options:
             config_text = cls._describe_configuration_text()
             if cls.__doc__ is None:
-                cls.__doc__ = f"Stage {cls.name}\n\nConfiguration Parameters:\n{config_text}"
+                cls.__doc__ = f"Stage {cls.name}\n\nParameters\n----------\n{config_text}"
             else:
                 # strip any existing configuration text from parent classes that is at the end of the doctring
-                cls.__doc__ = cls.__doc__.split("Configuration Parameters:")[0]
-                cls.__doc__ += f"\n\nConfiguration Parameters:\n{config_text}"
+                cls.__doc__ = cls.__doc__.split("Parameters")[0]
+                cls.__doc__ += f"\n\nParameters\n----------\n{config_text}"
 
         # Register the class
         if stage_is_complete:
@@ -436,24 +436,24 @@ class PipelineStage:
         s = []
         if cls.config_options is None:
             return "<This class has no configuration options>"
+
         for name, val in cls.config_options.items():
+            if isinstance(val, StageConfig):
+                val = val[name]
             if isinstance(val, StageParameter):
-                if val.required:
-                    if val.dtype is None:
-                        txt = f"[type not specified]: {val._help} (required)"
-                    else:
-                        txt = f"[{val.dtype.__name__}]: {val._help}  (required)"
-                else:
-                    if val.dtype is None:
-                        txt = f"[type not specified]: {val._help} (default={val.default})"
-                    else:
-                        txt = f"[{val.dtype.__name__}]: {val._help} (default={val.default})"
+                s.append(f"{name}: {val.numpy_style_help_text()}")
             elif isinstance(val, type):
-                txt = f"[{val.__name__}]: (required)"
+                s.append(f"{name}: {val.__name__}] (required)")
             else:
-                txt = f"[{type(val).__name__}]: (default={val})"
-            s.append(f"{name} {txt} ")
-        return '\n'.join(s)
+                s.append(f"{name}: {type(val).__name__}] (default={val})")
+
+        for input_ in cls.inputs:
+            s.append(f"{input_[0]}: {input_[1].__name__} (INPUT)")
+        for output_ in cls.outputs:
+            s.append(f"{output_[0]}: {output_[1].__name__} (OUTPUT)")
+
+        return '\n\n'.join(s)
+
 
     @classmethod
     def usage(cls):  # pragma: no cover

{ceci-2.1 → ceci-2.2}/ceci.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ceci
-Version: 2.1
+Version: 2.2
 Summary: Lightweight pipeline engine for LSST DESC
 Author: Joe Zuntz
 License: BSD 3-Clause License

{ceci-2.1 → ceci-2.2}/tests/test_stage.py

@@ -149,7 +149,7 @@ def test_parameter():
     assert stage_1.config.b == 'puffins are not extinct?'
     assert 1 in stage_1.config.c
     assert len(stage_1.config.d) == 0
-    assert "c [list]: a list (default=[1, 2, 3])" in stage_1.__doc__
+    assert "[list] default=[1, 2, 3]" in stage_1.__doc__
     stage_1.describe_configuration()
     
     cmd = "TestStage", "--a", "6", "--b", "puffins are not extinct?", "--inp", "dummy"