starbash 0.3.1__tar.gz → 0.3.2__tar.gz

{starbash-0.3.1 → starbash-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starbash
-Version: 0.3.1
+Version: 0.3.2
 Summary: A tool for automating/standardizing/sharing astrophotography workflows.
 License-File: LICENSE
 Author: Kevin Hester

{starbash-0.3.1 → starbash-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "starbash"
-version = "0.3.1"
+version = "0.3.2"
 description = "A tool for automating/standardizing/sharing astrophotography workflows."
 authors = ["Kevin Hester <kevinh@geeksville.com>"]
 readme = "README.md"

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/app.py

@@ -15,6 +15,7 @@ from rich.progress import track
 from toml_repo import Repo, RepoManager, get_config_suffix
 
 import starbash
+from starbash import doit_types
 from starbash.aliases import (
     Aliases,
     get_aliases,
@@ -185,6 +186,9 @@ class Starbash:
 
         # Load app defaults and initialize the repository manager
         self._init_repos()
+        doit_types.configure_max_contexts(
+            self.user_repo.get("config.max_contexts", doit_types.DEFAULT_MAX_CONTEXTS)
+        )
         self._init_analytics(cmd)  # after init repos so we have user prefs
 
         app_version = version("starbash")

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/defaults/starbash.toml

@@ -91,6 +91,14 @@ model.long = "Optolong 2-inch L-Extreme Dual Band 7nm Ha+Oiii Filter"
 url.info = "https://www.highpointscientific.com/optolong-2-inch-l-extreme-dual-band-7nm-ha-oiii-filter-lxt-200"
 subtype = "HaOiii"
 
+[[equipment]]
+type = "filter"
+vendor = "Agena"
+fits.filter = "Tri" # must match the FITS header FILTER field for this filter if you want auto matching.
+model.short = "ant-tri"
+model.long = "Agena Tri-Band RGB Ultra Filter"
+url.info = "https://agenaastro.com/antlia-triband-rgb-ultra-filter-2-mounted.html"
+
 [[equipment]]
 type = "mount"
 vendor = "Juwei"
@@ -138,7 +146,9 @@ HaOiii = ["HaOiii", "HaO3", "Duo"]
 # Astro is the name Dwarf3 uses for their IR cut filter
 # VIS is the name Dwarf3 uses for their 'clear' filter
 # IRCUT/LP are the names Seestar uses for their filters
-None = ["None", "Astro", "IRCUT", "LP", "VIS"]
+# Tri is a common name for a tri-band light pollution filter (LPF)
+
+None = ["None", "Astro", "IRCUT", "LP", "VIS", "Tri"]
 
 camera_osc = ["OSC", "ZWO ASI2600MC Duo"]
 camera_seestar = ["Seestar", "Seestar S50", "Seestar S30", "Seestar S30 Pro"]
@@ -157,7 +167,10 @@ camera_seestar = ["Seestar", "Seestar S50", "Seestar S30", "Seestar S30 Pro"]
 # http://www.antliafilter.com/pd.jsp?fromColId=2&id=160#_pp=2_671
 SiiHb = ["SiiHb", "S2Hb"]
 
+[config]
+# Set this lower if you need to reduce disk usage. If omitted, the default is 2 contexts.
+max_contexts = 2
 
 # List the initial search path to find other repos, when the app launches it always first loads this sb.toml file and this file should list all of the
 # standard default repo locations.  When searching repos, repos listed LAST have precedence, so target file can override the root processing defaults,
-# then the user prefs, then a live github URL or whatever
+# then the user prefs, then a live github URL or whatever

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/doit.py

@@ -22,7 +22,7 @@ from toml_repo import Repo
 from starbash import InputDef
 from starbash.database import ImageRow
 from starbash.doit_types import TaskDict
-from starbash.exception import UserHandledError
+from starbash.exception import FilesystemUnavailableError, UserHandledError
 from starbash.os import symlink_or_copy
 from starbash.paths import get_user_cache_dir
 from starbash.tool.base import Tool
@@ -257,7 +257,9 @@ def merge_to(base_name: str, fi: FileInfo) -> None:
                             indexed_ids = []
                             break
                         indexed_ids.append(source_ids[source_index - 1])
-                    if len(indexed_ids) == len(matching_files) and len(set(indexed_ids)) == len(indexed_ids):
+                    if len(indexed_ids) == len(matching_files) and len(set(indexed_ids)) == len(
+                        indexed_ids
+                    ):
                         resolved_source_ids = indexed_ids
                     else:
                         logging.warning(
@@ -269,7 +271,9 @@ def merge_to(base_name: str, fi: FileInfo) -> None:
                         )
 
             if resolved_source_ids is not None:
-                for matching_file, source_id in zip(matching_files, resolved_source_ids, strict=True):
+                for matching_file, source_id in zip(
+                    matching_files, resolved_source_ids, strict=True
+                ):
                     source_path = Path(matching_file)
                     collected_files.append(source_path)
                     provenance[source_path.name] = source_id
@@ -288,7 +292,7 @@ def merge_to(base_name: str, fi: FileInfo) -> None:
     ):
         dest_name = f"{base_name}_{index:05d}.fits"
         dest_path = output_dir / dest_name
-        #logging.debug(f"Linking {source_file} to {dest_path}")
+        # logging.debug(f"Linking {source_file} to {dest_path}")
         symlink_or_copy(str(source_file), str(dest_path))
 
     merged_provenance = {
@@ -355,6 +359,17 @@ class ToolAction(BaseAction):
                 self.result = self.tool.run(
                     self.commands, context=context, cwd=self.cwd, log_out=logfile, **self.parameters
                 )
+        except OSError as e:
+            error = FilesystemUnavailableError(
+                f"running {self.tool.name} for task '{self.task.name}'", e
+            )
+            logging.error("%s (%s)", error, e)
+            self.task.meta["exception"] = error
+            return TaskFailed(str(error))
+        except FilesystemUnavailableError as e:
+            logging.error("%s", e)
+            self.task.meta["exception"] = e
+            return TaskFailed(str(e))
         except Exception as e:
             # We pass back any exceptions in task.meta - so that our ConsoleReporter can pick them up (doit normally strips exceptions)
             self.task.meta["exception"] = e
@@ -422,6 +437,12 @@ class ProcessingResult:
                 # General error from user misconfiguration or tools - not a bug in our code
                 logging.error(f"Skipping run due to: {e}")
                 self.notes = str(e)
+            elif isinstance(e, OSError):
+                self.notes = (
+                    "The filesystem became unavailable during processing. "
+                    "Check that the drive or network mount is connected, then retry."
+                )
+                logging.error("Filesystem unavailable during processing: %s", e)
             else:
                 # Unexpected exception - log it and re-raise
                 logging.exception("Unexpected error during processing:")
@@ -562,7 +583,7 @@ class StarbashDoit(TaskLoader2):
             "verbosity": 2,
             "dep_file": dep_file,
             "reporter": MyReporter,
-            "backend": "dbm", # the json backend is slow and buggy, use dbm instead
+            "backend": "dbm",  # the json backend is slow and buggy, use dbm instead
         }
 
     def load_tasks(self, cmd: Any, pos_args: Any) -> list[Task]:

starbash-0.3.2/src/starbash/doit_types.py

@@ -0,0 +1,85 @@
+"""Shared types and utilities for doit processing to avoid circular imports."""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+from starbash.paths import get_user_cache_dir
+
+type TaskDict = dict[str, Any]  # a doit task dictionary
+
+DEFAULT_MAX_CONTEXTS = 2
+max_contexts = DEFAULT_MAX_CONTEXTS
+
+
+def configure_max_contexts(value: Any) -> None:
+    """Set the maximum number of processing contexts from user configuration.
+
+    Invalid values are ignored so a malformed preference cannot prevent
+    Starbash from starting.
+    """
+    global max_contexts
+
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        logging.warning(
+            "Ignoring invalid max_contexts preference %r; using %d.",
+            value,
+            DEFAULT_MAX_CONTEXTS,
+        )
+        max_contexts = DEFAULT_MAX_CONTEXTS
+        return
+
+    max_contexts = value
+
+
+def get_processing_dir() -> Path:
+    """Get the base directory for processing contexts."""
+    cache_dir = get_user_cache_dir()
+    processing_dir = cache_dir / "processing"
+    processing_dir.mkdir(parents=True, exist_ok=True)
+    return processing_dir
+
+
+def cleanup_old_contexts() -> None:
+    """Remove oldest context directories if we exceed max_contexts."""
+    processing_dir = get_processing_dir()
+    logging.debug(f"Removing old processing contexts in: {processing_dir}")
+    if not processing_dir.exists():
+        return
+
+    # Safety guard: if we're running under pytest but the processing dir is NOT a test
+    # override (no explicit cache override and no STARBASH_CACHE_DIR), refuse to delete
+    # anything. A leaked test global (e.g. a small max_contexts from a test config) must
+    # never prune the real user's cache.
+    from starbash import paths
+
+    if os.environ.get("PYTEST_CURRENT_TEST") is not None:
+        using_override = (
+            paths._override_cache_dir is not None or os.getenv("STARBASH_CACHE_DIR")
+        )
+        if not using_override:
+            logging.warning(
+                "Refusing to clean processing contexts during tests without a cache override: %s",
+                processing_dir,
+            )
+            return
+
+    # Get all subdirectories in processing_dir
+    contexts = [d for d in processing_dir.iterdir() if d.is_dir()]
+
+    # If we have more than max_contexts, delete the oldest ones
+    if len(contexts) > max_contexts:
+        # Sort by modification time (oldest first)
+        contexts.sort(key=lambda d: d.stat().st_mtime)
+
+        # Calculate how many to delete
+        num_to_delete = len(contexts) - max_contexts
+
+        # Delete the oldest directories
+        for context_dir in contexts[:num_to_delete]:
+            logging.debug(f"Removing old processing context: {context_dir}")
+            shutil.rmtree(context_dir, ignore_errors=True)

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/exception.py

@@ -25,6 +25,17 @@ class UserHandledError(NonSoftwareError):
         return False
 
 
+class FilesystemUnavailableError(UserHandledError):
+    """Raised when a filesystem disappears while processing is in progress."""
+
+    def __init__(self, operation: str, cause: OSError) -> None:
+        super().__init__(
+            f"The filesystem became unavailable while {operation}. "
+            "Check that the drive or network mount is connected, then retry."
+        )
+        self.cause = cause
+
+
 def raise_missing_repo(kind: str) -> NoReturn:
     """Raise a UserHandledError indicating that a repository of the given kind is missing."""
     raise UserHandledError(
@@ -54,6 +65,7 @@ class NoSuitableMastersException(NonFatalException):
 
 __all__ = [
     "UserHandledError",
+    "FilesystemUnavailableError",
     "NonFatalException",
     "NotEnoughFilesError",
     "NoSuitableMastersException",

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/filtering.py

@@ -30,6 +30,10 @@ def _apply_filter(requires: RequireDef, candidates: list[ImageRow]) -> list[Imag
 
     kind = get_safe(requires, "kind")
     value = requires.get("value")  # value is optional for some kinds
+    # Optional `invert` negates the boolean match so non-matching candidates are
+    # kept (e.g. select sessions whose filter is NOT narrowband). Only meaningful
+    # for boolean-match kinds (metadata/camera/unprocessed/filename), not min_count.
+    invert = bool(requires.get("invert", False))
 
     # Stage 1: Filter candidates using kind-specific filter functions
     def _filter_metadata(metadata: Metadata) -> bool:
@@ -95,7 +99,10 @@ def _apply_filter(requires: RequireDef, candidates: list[ImageRow]) -> list[Imag
         raise ValueError(f"Unknown requires kind: {kind}")
 
     # Apply the filter function to all candidates
-    filtered_candidates = [img for img in candidates if filter_func(img)]
+    if invert:
+        filtered_candidates = [img for img in candidates if not filter_func(img)]
+    else:
+        filtered_candidates = [img for img in candidates if filter_func(img)]
 
     # Stage 2: Handle min_count check after filtering
     if kind == "min_count":

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/fits.py

@@ -8,11 +8,34 @@ from astropy.io import fits
 __all__ = ["read_dimensions", "read_fits_header"]
 
 
+def _json_safe(value: Any) -> Any:
+    """Convert an astropy header value into a plain JSON-serializable Python value.
+
+    Commentary cards (COMMENT/HISTORY/blank keyword) come back as
+    ``_HeaderCommentaryCards`` (a list-like of strings) and other cards can be
+    ``bool``, numpy scalars etc. None of those serialize via ``json.dumps``.
+    """
+    # Commentary cards: list-like container of strings; join into one string
+    if isinstance(value, fits.header._HeaderCommentaryCards):
+        return "\n".join(value)
+    if isinstance(value, (list, tuple)):
+        return [_json_safe(v) for v in value]
+    if isinstance(value, bool):  # must precede int check (bool is an int subclass)
+        return value
+    if isinstance(value, (int, float, str)) or value is None:
+        return value
+    # numpy scalars and anything else: fall back to str()
+    return str(value)
+
+
 def read_fits_header(file_path: str | Path) -> dict[str, Any]:
-    """Read and return the primary FITS header from ``file_path``."""
+    """Read and return the primary FITS header from ``file_path``.
+
+    All values are converted to plain JSON-serializable Python types.
+    """
     with fits.open(str(file_path), memmap=False) as hdul:
         hdu0: Any = hdul[0]
-        return dict(hdu0.header)
+        return {key: _json_safe(value) for key, value in hdu0.header.items()}
 
 
 def read_dimensions(file_path: str | Path) -> tuple[int, int]:

{starbash-0.3.1 → starbash-0.3.2}/src/starbash/processing.py

@@ -16,7 +16,7 @@ from toml_repo import Repo
 from tomlkit.items import AoT
 
 import starbash
-from starbash import InputDef, OutputDef, RequireDef, StageDict
+from starbash import InputDef, Metadata, OutputDef, RequireDef, StageDict
 from starbash.aliases import get_aliases, normalize_target_name
 from starbash.app import Starbash
 from starbash.database import (
@@ -82,11 +82,7 @@ def _clone_context(context: dict[str, Any]) -> dict[str, Any]:
     Returns:
         A deep copy of the current context dictionary.
     """
-    shared = {
-        key: context[key]
-        for key in ["session", "update_image_metadata"]
-        if key in context
-    }
+    shared = {key: context[key] for key in ["session", "update_image_metadata"] if key in context}
     r = copy.deepcopy({key: value for key, value in context.items() if key not in shared})
 
     # A few fields (if populated) we want SHARED between all contexts, so that if two contexts were initially pointing
@@ -197,7 +193,12 @@ class Processing(ProcessingLike):
     def __enter__(self) -> "Processing":
         return self
 
-    def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None) -> bool:
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: types.TracebackType | None,
+    ) -> bool:
         self.close()
         return False
 
@@ -637,9 +638,11 @@ class Processing(ProcessingLike):
                 try:
                     script = source.read(script_filename)
                     try:
-                        script_filename = source.resolve_path(script_filename) # Try to let the tool give the full filepath in error messages
+                        script_filename = source.resolve_path(
+                            script_filename
+                        )  # Try to let the tool give the full filepath in error messages
                     except Exception:
-                        pass # some repos might not be on a local disk.  In that case just use the base name
+                        pass  # some repos might not be on a local disk.  In that case just use the base name
                     tool_parameters["script_file"] = str(script_filename)
 
                 except OSError as e:
@@ -691,9 +694,18 @@ class Processing(ProcessingLike):
             return None
 
         after = get_safe(input_with_after, "after")
-        prior_task_name = self._get_unique_task_name(
-            after
-        )  # find the right task for our stage and multiplex
+        # ``after`` is a regex over task/stage names. Preserve it as a regex
+        # instead of passing it through ``_get_unique_task_name()``, which is
+        # intended for literal task names and would turn e.g.
+        # ``stack_(single|dual)_duo`` into a non-matching literal prefix.
+        prior_task_name = self._get_unique_task_name(after)
+        # ``_get_unique_task_name`` appends target/session suffixes after the
+        # regex. Wrap the dependency expression so the suffix remains outside
+        # the alternation (``stack_(single|dual)_duo_sh2126`` must match both
+        # concrete stage names).
+        if "(" in after or "[" in after or "|" in after:
+            suffix = prior_task_name[len(after) :]
+            prior_task_name = f"(?:{after}){suffix}"
 
         # Compile the prior_task_name into a regex pattern for prefix matching.
         # The pattern from TOML may contain wildcards like "light.*" which should match
@@ -970,11 +982,17 @@ class Processing(ProcessingLike):
             self.context.pop("stage_input", None)
             self.context.pop("multiplex_index", None)
 
-    def _with_defaults(self, img: ImageRow) -> ImageRow:
+    def _with_defaults(self, img: ImageRow, defaults: Metadata | None = None) -> ImageRow:
         """Try to provide missing metadata for image rows.  Some imagerows are 'sparse'
         with just a filename and minor other info.  In that case try to assume the metadata matches
-        the input metadata for this single pipeline of images."""
-        r = self.context.get("default_metadata", {}).copy()
+        the input metadata for this single pipeline of images.
+
+        Args:
+            img: The image row to complete.
+            defaults: Metadata belonging to the task that produced ``img``.
+                When omitted, use the current processing context.
+        """
+        r = (defaults if defaults is not None else self.context.get("default_metadata", {})).copy()
 
         # values from the passed in img override our defaults
         for key, value in img.items():
@@ -1027,12 +1045,11 @@ class Processing(ProcessingLike):
             )
         ]
         if not prior_tasks:
-            raise NotEnoughFilesError(
-                "All prior stages for this input were excluded", []
-            )
+            raise NotEnoughFilesError("All prior stages for this input were excluded", [])
 
         # Collect all image rows from prior stage outputs
         child_exception: Exception | None = None
+        matching_input_rows: list[ImageRow] = []
 
         # `filename` requires match the actual files we consume (the prior stage's
         # OUTPUT names), which differ from its inputs; every other requires kind
@@ -1040,9 +1057,15 @@ class Processing(ProcessingLike):
         # INPUT rows. So split them and apply each to the correct row set.
         all_requires: list[RequireDef] = input.get("requires", [])
         filename_requires = [r for r in all_requires if r.get("kind") == "filename"]
+        min_count_requires = [r for r in all_requires if r.get("kind") == "min_count"]
         gating_input: InputDef = (
-            {**input, "requires": [r for r in all_requires if r.get("kind") != "filename"]}
-            if filename_requires
+            {
+                **input,
+                "requires": [
+                    r for r in all_requires if r.get("kind") not in ("filename", "min_count")
+                ],
+            }
+            if filename_requires or min_count_requires
             else input
         )
 
@@ -1055,12 +1078,18 @@ class Processing(ProcessingLike):
             for _input_type, file_info in task_inputs.items():
                 if isinstance(file_info, FileInfo) and file_info.image_rows:
                     images = file_info.image_rows
-                    images = [self._with_defaults(img) for img in images]
+                    # A multiplexed upstream stage has one task per channel.
+                    # Use that task's metadata, rather than the current
+                    # context's metadata (which is usually copied from the
+                    # last multiplexed task), when applying metadata filters.
+                    task_defaults = task_context.get("default_metadata", {})
+                    images = [self._with_defaults(img, task_defaults) for img in images]
                     try:
                         task_filtered_input = filter_by_requires(gating_input, images)
                         if (
                             task_filtered_input
                         ):  # This task had matching inputs for us, so therefore we want its outputs
+                            matching_input_rows.extend(task_filtered_input)
                             task_output = task_context.get("output")
                             if (
                                 task_output
@@ -1081,16 +1110,30 @@ class Processing(ProcessingLike):
                         # just because one prior task doesn't have what we need, we shouldn't stop looking
                         logging.debug(f"Prior task '{task['name']}' skipped, still looking... {e}")
 
-        if child_exception and len(image_rows) == 0:
+        if child_exception and len(image_rows) == 0 and not input.get("optional", False):
             # we failed on every child, give up
             raise child_exception
 
-        if filename_requires and len(image_rows) == 0:
+        if filename_requires and len(image_rows) == 0 and not input.get("optional", False):
             # Prior outputs existed but none matched the filename filter; skip this stage.
             raise NotEnoughFilesError(
                 "No prior stage outputs matched the input filename filter", []
             )
 
+        if min_count_requires:
+            # Apply min_count to this input group's aggregate, rather than to
+            # each multiplexed upstream task.  This keeps the requirement
+            # scoped to one [[stages.inputs]] block, so separate input groups
+            # can have independent counts.
+            try:
+                filter_by_requires({"requires": min_count_requires}, matching_input_rows)
+            except NotEnoughFilesError:
+                if not input.get("optional", False):
+                    raise
+
+        if not image_rows and input.get("optional", False):
+            return FileInfo(image_rows=[], definition=input)
+
         sequence_provenance: dict[str, list[int]] = {}
         provenance: dict[str, int] = {}
         for task in prior_tasks:
@@ -1277,13 +1320,9 @@ class Processing(ProcessingLike):
                 definition=input,
             )
             source_images = [image for image in images if image.get("id") is not None]
-            fi.sequence_provenance = {
-                f"{fi.base}_.seq": [image["id"] for image in source_images]
-            }
+            fi.sequence_provenance = {f"{fi.base}_.seq": [image["id"] for image in source_images]}
             fi.provenance = {
-                image["path"]: image["id"]
-                for image in source_images
-                if image.get("path")
+                image["path"]: image["id"] for image in source_images if image.get("path")
             }
             ci[imagetyp] = fi
 
@@ -1460,9 +1499,7 @@ class Processing(ProcessingLike):
                     continue
                 sequence_provenance[filename] = source_ids.copy()
                 stem = (
-                    filename[: -len("_.seq")]
-                    if filename.endswith("_.seq")
-                    else Path(filename).stem
+                    filename[: -len("_.seq")] if filename.endswith("_.seq") else Path(filename).stem
                 )
                 generated_provenance.update(
                     {