starbash 0.1.11__py3-none-any.whl → 0.1.15__py3-none-any.whl

starbash/app.py

@@ -1,34 +1,18 @@
-import cmd
 import logging
-from importlib import resources
-import os
+import shutil
+from datetime import datetime
 from pathlib import Path
-import tempfile
-import typer
-import tomlkit
-from tomlkit.toml_file import TOMLFile
-import glob
 from typing import Any
+
+import rich.console
+import typer
 from astropy.io import fits
-import itertools
-from rich.progress import track
 from rich.logging import RichHandler
-import shutil
-from datetime import datetime
-import rich.console
-import copy
+from rich.progress import track
 
 import starbash
-from starbash import console, _is_test_env, to_shortdate
-from starbash.aliases import Aliases
-from starbash.database import Database, SessionRow, ImageRow, get_column_name
-from repo import Repo, repo, repo_suffix
-from starbash.toml import toml_from_template
-from starbash.tool import Tool, expand_context, expand_context_unsafe
-from repo import RepoManager
-from starbash.tool import tools
-from starbash.paths import get_user_config_dir, get_user_data_dir
-from starbash.selection import Selection, where_tuple
+from repo import Repo, RepoManager, repo_suffix
+from starbash.aliases import Aliases, normalize_target_name
 from starbash.analytics import (
     NopAnalytics,
     analytics_exception,
@@ -36,17 +20,30 @@ from starbash.analytics import (
     analytics_shutdown,
     analytics_start_transaction,
 )
-
-# Type aliases for better documentation
+from starbash.check_version import check_version
+from starbash.database import (
+    Database,
+    ImageRow,
+    SearchCondition,
+    SessionRow,
+    get_column_name,
+)
+from starbash.paths import get_user_config_dir, get_user_config_path
+from starbash.selection import Selection, build_search_conditions
+from starbash.toml import toml_from_template
+from starbash.tool import preflight_tools
 
 
-def setup_logging(stderr: bool = False):
+def setup_logging(console: rich.console.Console):
     """
     Configures basic logging.
     """
-    console = rich.console.Console(stderr=stderr)
+    from starbash import _is_test_env  # Lazy import to avoid circular dependency
+
     handlers = (
-        [RichHandler(console=console, rich_tracebacks=True)] if not _is_test_env else []
+        [RichHandler(console=console, rich_tracebacks=True, markup=True)]
+        if not _is_test_env
+        else []
     )
     logging.basicConfig(
         level=starbash.log_filter_level,  # use the global log filter level
@@ -56,12 +53,6 @@ def setup_logging(stderr: bool = False):
     )
 
 
-def get_user_config_path() -> Path:
-    """Returns the path to the user config file."""
-    config_dir = get_user_config_dir()
-    return config_dir / repo_suffix
-
-
 def create_user() -> Path:
     """Create user directories if they don't exist yet."""
     path = get_user_config_path()
@@ -77,6 +68,7 @@ def copy_images_to_dir(images: list[ImageRow], output_dir: Path) -> None:
     This function requires that "abspath" already be populated in each ImageRow.  Normally
     the caller does this by calling Starbash._add_image_abspath() on the image.
     """
+    from starbash import console  # Lazy import to avoid circular dependency
 
     # Export images
     console.print(f"[cyan]Exporting {len(images)} images to {output_dir}...[/cyan]")
@@ -115,7 +107,7 @@ def copy_images_to_dir(images: list[ImageRow], output_dir: Path) -> None:
                 error_count += 1
 
     # Print summary
-    console.print(f"[green]Export complete![/green]")
+    console.print("[green]Export complete![/green]")
     if linked_count > 0:
         console.print(f"  Linked: {linked_count} files")
     if copied_count > 0:
@@ -124,33 +116,6 @@ def copy_images_to_dir(images: list[ImageRow], output_dir: Path) -> None:
         console.print(f"  [red]Errors: {error_count} files[/red]")
 
 
-class ProcessingContext(tempfile.TemporaryDirectory):
-    """For processing a set of sessions for a particular target.
-
-    Keeps a shared temporary directory for intermediate files.  We expose the path to that
-    directory in context["process_dir"].
-    """
-
-    def __init__(self, starbash: "Starbash"):
-        super().__init__(prefix="sbprocessing_")
-        self.sb = starbash
-        logging.debug(f"Created processing context at {self.name}")
-
-        self.sb.init_context()
-        self.sb.context["process_dir"] = self.name
-
-    def __enter__(self) -> "ProcessingContext":
-        return super().__enter__()
-
-    def __exit__(self, exc_type, exc_value, traceback) -> None:
-        logging.debug(f"Cleaning up processing context at {self.name}")
-
-        # unregister our process dir
-        self.sb.context.pop("process_dir", None)
-
-        super().__exit__(exc_type, exc_value, traceback)
-
-
 class Starbash:
     """The main Starbash application class."""
 
@@ -158,24 +123,41 @@ class Starbash:
         """
         Initializes the Starbash application by loading configurations
         and setting up the repository manager.
+
+        Args:
+            cmd (str): The command name or identifier for the current Starbash session.
+            stderr_logging (bool): Whether to enable logging to stderr.
+            no_progress (bool): Whether to disable the (asynchronous) progress display (because it breaks typer.ask)
         """
-        setup_logging(stderr=stderr_logging)
+        from starbash import _is_test_env  # Lazy import to avoid circular dependency
+
+        # It is important to disable fancy colors and line wrapping if running under test - because
+        # those tests will be string parsing our output.
+        console = rich.console.Console(
+            force_terminal=False if _is_test_env else None,
+            width=999999 if _is_test_env else None,  # Disable line wrapping in tests
+            stderr=stderr_logging,
+        )
+
+        starbash.console = console  # Update the global console to use the progress version
+
+        setup_logging(starbash.console)
         logging.info("Starbash starting...")
 
         # Load app defaults and initialize the repository manager
         self._init_repos()
-        self._init_analytics(cmd)
+        self._init_analytics(cmd)  # after init repos so we have user prefs
+        check_version()
         self._init_aliases()
 
-        logging.info(
-            f"Repo manager initialized with {len(self.repo_manager.repos)} repos."
-        )
+        logging.info(f"Repo manager initialized with {len(self.repo_manager.repos)} repos.")
         # self.repo_manager.dump()
 
         self._db = None  # Lazy initialization - only create when accessed
 
         # Initialize selection state (stored in user config repo)
         self.selection = Selection(self.user_repo)
+        preflight_tools()
 
     def _init_repos(self) -> None:
         """Initialize all repositories managed by the RepoManager."""
@@ -189,9 +171,7 @@ class Starbash:
         self.analytics = NopAnalytics()
         if self.user_repo.get("analytics.enabled", True):
             include_user = self.user_repo.get("analytics.include_user", False)
-            user_email = (
-                self.user_repo.get("user.email", None) if include_user else None
-            )
+            user_email = self.user_repo.get("user.email", None) if include_user else None
             if user_email is not None:
                 user_email = str(user_email)
             analytics_setup(allowed=True, user_email=user_email)
@@ -247,8 +227,9 @@ class Starbash:
         self.close()
         return handled
 
-    def _add_session(self, image_doc_id: int, header: dict) -> None:
+    def _add_session(self, header: dict) -> None:
         """We just added a new image, create or update its session entry as needed."""
+        image_doc_id: int = header[Database.ID_KEY]  # this key is required to exist
         image_type = header.get(Database.IMAGETYP_KEY)
         date = header.get(Database.DATE_OBS_KEY)
         if not date or not image_type:
@@ -286,9 +267,47 @@ class Starbash:
             session = self.db.get_session(new)
             self.db.upsert_session(new, existing=session)
 
-    def guess_sessions(
-        self, ref_session: SessionRow, want_type: str
-    ) -> list[SessionRow]:
+    def add_local_repo(self, path: str, repo_type: str | None = None) -> None:
+        """Add a local repository located at the specified path.  If necessary toml config files
+        will be created at the root of the repository."""
+
+        p = Path(path)
+        console = starbash.console
+
+        repo_toml = p / repo_suffix  # the starbash.toml file at the root of the repo
+        if repo_toml.exists():
+            logging.warning("Using existing repository config file: %s", repo_toml)
+        else:
+            if repo_type:
+                console.print(f"Creating {repo_type} repository: {p}")
+                p.mkdir(parents=True, exist_ok=True)
+
+                toml_from_template(
+                    f"repo/{repo_type}",
+                    p / repo_suffix,
+                    overrides={
+                        "REPO_TYPE": repo_type,
+                        "REPO_PATH": str(p),
+                    },
+                )
+            else:
+                # No type specified, therefore (for now) assume we are just using this as an input
+                # repo (and it must exist)
+                if not p.exists():
+                    console.print(f"[red]Error: Repo path does not exist: {p}[/red]")
+                    raise typer.Exit(code=1)
+
+        console.print(f"Adding repository: {p}")
+
+        repo = self.user_repo.add_repo_ref(p)
+        if repo:
+            self.reindex_repo(repo)
+
+            # we don't yet always write default config files at roots of repos, but it would be easy to add here
+            # r.write_config()
+            self.user_repo.write_config()
+
+    def guess_sessions(self, ref_session: SessionRow, want_type: str) -> list[SessionRow]:
         """Given a particular session type (i.e. FLAT or BIAS etc...) and an
         existing session (which is assumed to generally be a LIGHT frame based session):
 
@@ -320,13 +339,11 @@ class Starbash:
         }
 
         # For FLAT frames, filter must match the reference session
-        if want_type.upper() == "FLAT":
-            conditions[Database.FILTER_KEY] = ref_session[
-                get_column_name(Database.FILTER_KEY)
-            ]
+        if want_type.lower() == "flat":
+            conditions[Database.FILTER_KEY] = ref_session[get_column_name(Database.FILTER_KEY)]
 
         # Search for candidate sessions
-        candidates = self.db.search_session(where_tuple(conditions))
+        candidates = self.db.search_session(build_search_conditions(conditions))
 
         return self.score_candidates(candidates, ref_session)
 
@@ -395,15 +412,13 @@ class Starbash:
                         # Using 7-day half-life
                         score += 100 * (2.718 ** (-time_delta / (7 * 86400)))
                     except (ValueError, TypeError):
-                        logging.warning(f"Malformed date - ignoring entry")
+                        logging.warning("Malformed date - ignoring entry")
 
                 scored_candidates.append((score, candidate))
 
             except (AssertionError, KeyError) as e:
                 # If we can't get the session image, log and skip this candidate
-                logging.warning(
-                    f"Could not score candidate session {candidate.get('id')}: {e}"
-                )
+                logging.warning(f"Could not score candidate session {candidate.get('id')}: {e}")
                 continue
 
         # Sort by score (highest first)
@@ -411,10 +426,13 @@ class Starbash:
 
         return [candidate for _, candidate in scored_candidates]
 
-    def search_session(self) -> list[SessionRow]:
+    def search_session(self, conditions: list[SearchCondition] | None = None) -> list[SessionRow]:
         """Search for sessions, optionally filtered by the current selection."""
         # Get query conditions from selection
-        conditions = self.selection.get_query_conditions()
+        if conditions is None:
+            conditions = self.selection.get_query_conditions()
+
+        self.add_filter_not_masters(conditions)  # we never return processed masters as sessions
         return self.db.search_session(conditions)
 
     def _add_image_abspath(self, image: ImageRow) -> ImageRow:
@@ -445,15 +463,9 @@ class Starbash:
         from starbash.database import SearchCondition
 
         images = self.db.search_image(
-            [
-                SearchCondition(
-                    "i.id", "=", session[get_column_name(Database.IMAGE_DOC_KEY)]
-                )
-            ]
+            [SearchCondition("i.id", "=", session[get_column_name(Database.IMAGE_DOC_KEY)])]
         )
-        assert (
-            len(images) == 1
-        ), f"Expected exactly one reference for session, found {len(images)}"
+        assert len(images) == 1, f"Expected exactly one reference for session, found {len(images)}"
         return self._add_image_abspath(images[0])
 
     def get_master_images(
@@ -487,9 +499,34 @@ class Starbash:
             search_conditions.append(SearchCondition("i.imagetyp", "=", imagetyp))
 
         images = self.db.search_image(search_conditions)
+
+        # FIXME - move this into a general filter function
+        # For flat frames, filter images based on matching reference_session filter
+        if reference_session and imagetyp and self.aliases.normalize(imagetyp) == "flat":
+            ref_filter = self.aliases.normalize(
+                reference_session.get(get_column_name(Database.FILTER_KEY), "None")
+            )
+            if ref_filter:
+                # Filter images to only those with matching filter in metadata
+                filtered_images = []
+                for img in images:
+                    img_filter = img.get(Database.FILTER_KEY, "None")
+                    if img_filter == ref_filter:
+                        filtered_images.append(img)
+                images = filtered_images
+
         return images
 
-    def get_session_images(self, session: SessionRow) -> list[ImageRow]:
+    def add_filter_not_masters(self, conditions: list[SearchCondition]) -> None:
+        """Add conditions to filter out master and processed repos from image searches."""
+        master_repo = self.repo_manager.get_repo_by_kind("master")
+        if master_repo is not None:
+            conditions.append(SearchCondition("r.url", "<>", master_repo.url))
+        processed_repo = self.repo_manager.get_repo_by_kind("processed")
+        if processed_repo is not None:
+            conditions.append(SearchCondition("r.url", "<>", processed_repo.url))
+
+    def get_session_images(self, session: SessionRow, processed_ok: bool = False) -> list[ImageRow]:
         """
         Get all images belonging to a specific session.
 
@@ -500,6 +537,9 @@ class Starbash:
         Args:
             session_id: The database ID of the session
 
+            processed_ok: If True, include images which were processed by apps (i.e. stacked or other procesing)
+            Normally image pipelines don't want to accidentially consume those files.
+
         Returns:
             List of image records (dictionaries with path, metadata, etc.)
             Returns empty list if session not found or has no images.
@@ -513,23 +553,17 @@ class Starbash:
         # Note: We need to search JSON metadata for FILTER, IMAGETYP, OBJECT, TELESCOP
         # since they're not indexed columns in the images table
         conditions = [
-            SearchCondition(
-                "i.date_obs", ">=", session[get_column_name(Database.START_KEY)]
-            ),
-            SearchCondition(
-                "i.date_obs", "<=", session[get_column_name(Database.END_KEY)]
-            ),
-            SearchCondition(
-                "i.imagetyp", "=", session[get_column_name(Database.IMAGETYP_KEY)]
-            ),
+            SearchCondition("i.date_obs", ">=", session[get_column_name(Database.START_KEY)]),
+            SearchCondition("i.date_obs", "<=", session[get_column_name(Database.END_KEY)]),
+            SearchCondition("i.imagetyp", "=", session[get_column_name(Database.IMAGETYP_KEY)]),
         ]
 
-        # we never want to return 'master' images as part of the session image paths
+        # Note: not needed here, because we filter this earlier - when building the
+        # list of candidate sessions.
+        # we never want to return 'master' or 'processed' images as part of the session image paths
         # (because we will be passing these tool siril or whatever to generate masters or
         # some other downstream image)
-        master_repo = self.repo_manager.get_repo_by_kind("master")
-        if master_repo is not None:
-            conditions.append(SearchCondition("r.url", "<>", master_repo.url))
+        # self.add_filter_not_masters(conditions)
 
         # Single query with indexed date conditions
         images = self.db.search_image(conditions)
@@ -537,22 +571,24 @@ class Starbash:
         # We no lognger filter by target(object) because it might not be set anyways
         filtered_images = []
         for img in images:
+            # "HISTORY" nodes are added by processing tools (Siril etc...), we never want to accidentally read those images
+            has_history = img.get("HISTORY")
+
+            # images that were stacked seem to always have a STACKCNT header set
+            is_stacked = img.get("STACKCNT")
+
             if (
-                img.get(Database.FILTER_KEY)
-                == session[get_column_name(Database.FILTER_KEY)]
+                img.get(Database.FILTER_KEY) == session[get_column_name(Database.FILTER_KEY)]
                 # and img.get(Database.OBJECT_KEY)
                 # == session[get_column_name(Database.OBJECT_KEY)]
                 and img.get(Database.TELESCOP_KEY)
                 == session[get_column_name(Database.TELESCOP_KEY)]
+                and (processed_ok or (not has_history and not is_stacked))
             ):
                 filtered_images.append(img)
 
         # Reconstruct absolute paths for all images
-        return (
-            [self._add_image_abspath(img) for img in filtered_images]
-            if filtered_images
-            else []
-        )
+        return [self._add_image_abspath(img) for img in filtered_images]
 
     def remove_repo_ref(self, url: str) -> None:
         """
@@ -570,7 +606,7 @@ class Starbash:
         repo_refs = self.user_repo.config.get("repo-ref")
 
         if not repo_refs:
-            raise ValueError(f"No repository references found in user configuration.")
+            raise ValueError("No repository references found in user configuration.")
 
         # Find and remove the matching repo-ref
         found = False
@@ -590,7 +626,7 @@ class Starbash:
         # Write the updated config
         self.user_repo.write_config()
 
-    def add_image_to_db(self, repo: Repo, f: Path, force: bool = False) -> None:
+    def add_image(self, repo: Repo, f: Path, force: bool = False) -> dict[str, Any] | None:
         """Read FITS header from file and add/update image entry in the database."""
 
         path = repo.get_path()
@@ -602,51 +638,65 @@ class Starbash:
         if config:
             whitelist = config.get("fits-whitelist", None)
 
-        try:
-            # Convert absolute path to relative path within repo
-            relative_path = f.relative_to(path)
-
-            found = self.db.get_image(repo.url, str(relative_path))
-
-            # for debugging sometimes we want to limit scanning to a single directory or file
-            # debug_target = "masters-raw/2025-09-09/DARK"
-            debug_target = None
-            if debug_target:
-                if str(relative_path).startswith(debug_target):
-                    logging.error("Debugging %s...", f)
-                    found = False
-                else:
-                    found = True  # skip processing
-                    force = False
-
-            if not found or force:
-                # Read and log the primary header (HDU 0)
-                with fits.open(str(f), memmap=False) as hdul:
-                    # convert headers to dict
-                    hdu0: Any = hdul[0]
-                    header = hdu0.header
-                    if type(header).__name__ == "Unknown":
-                        raise ValueError("FITS header has Unknown type: %s", f)
-
-                    items = header.items()
-                    headers = {}
-                    for key, value in items:
-                        if (not whitelist) or (key in whitelist):
-                            headers[key] = value
-                    logging.debug("Headers for %s: %s", f, headers)
-                    # Store relative path in database
-                    headers["path"] = str(relative_path)
-                    image_doc_id = self.db.upsert_image(headers, repo.url)
-
-                    if not found:
-                        # Update the session infos, but ONLY on first file scan
-                        # (otherwise invariants will get messed up)
-                        self._add_session(image_doc_id, header)
-
-        except Exception as e:
-            logging.warning("Failed to read FITS header for %s: %s", f, e)
-
-    def reindex_repo(self, repo: Repo, force: bool = False, subdir: str | None = None):
+        # Convert absolute path to relative path within repo
+        relative_path = f.relative_to(path)
+
+        found = self.db.get_image(repo.url, str(relative_path))
+
+        # for debugging sometimes we want to limit scanning to a single directory or file
+        # debug_target = "masters-raw/2025-09-09/DARK"
+        debug_target = None
+        if debug_target:
+            if str(relative_path).startswith(debug_target):
+                logging.error("Debugging %s...", f)
+                found = False
+            else:
+                found = True  # skip processing
+                force = False
+
+        if not found or force:
+            # Read and log the primary header (HDU 0)
+            with fits.open(str(f), memmap=False) as hdul:
+                # convert headers to dict
+                hdu0: Any = hdul[0]
+                header = hdu0.header
+                if type(header).__name__ == "Unknown":
+                    raise ValueError("FITS header has Unknown type: %s", f)
+
+                items = header.items()
+                headers = {}
+                for key, value in items:
+                    if (not whitelist) or (key in whitelist):
+                        headers[key] = value
+
+                # Some device software (old Asiair versions) fails to populate TELESCOP, in that case fall back to
+                # CREATOR (see doc/fits/malformedasimaster.txt for an example)
+                if Database.TELESCOP_KEY not in headers:
+                    creator = headers.get("CREATOR")
+                    if creator:
+                        headers[Database.TELESCOP_KEY] = creator
+
+                logging.debug("Headers for %s: %s", f, headers)
+
+                # Store relative path in database
+                headers["path"] = str(relative_path)
+                image_doc_id = self.db.upsert_image(headers, repo.url)
+                headers[Database.ID_KEY] = image_doc_id
+
+                if not found:
+                    return headers
+
+        return None
+
+    def add_image_and_session(self, repo: Repo, f: Path, force: bool = False) -> None:
+        """Read FITS header from file and add/update image entry in the database."""
+        headers = self.add_image(repo, f, force=force)
+        if headers:
+            # Update the session infos, but ONLY on first file scan
+            # (otherwise invariants will get messed up)
+            self._add_session(headers)
+
+    def reindex_repo(self, repo: Repo, subdir: str | None = None):
         """Reindex all repositories managed by the RepoManager."""
 
         # make sure this new repo is listed in the repos table
@@ -654,7 +704,8 @@ class Starbash:
 
         path = repo.get_path()
 
-        if path and repo.is_scheme("file") and repo.kind != "recipe":
+        repo_kind = repo.kind()
+        if path and repo.is_scheme("file") and repo_kind != "recipe":
             logging.debug("Reindexing %s...", repo.url)
 
             if subdir:
@@ -667,84 +718,39 @@ class Starbash:
                 description=f"Indexing {repo.url}...",
             ):
                 # progress.console.print(f"Indexing {f}...")
-                self.add_image_to_db(repo, f, force=force)
+                if repo_kind == "master":
+                    # for master repos we only add to the image table
+                    self.add_image(repo, f, force=True)
+                elif repo_kind == "processed":
+                    pass  # we never add processed images to our db
+                else:
+                    self.add_image_and_session(repo, f, force=starbash.force_regen)
 
-    def reindex_repos(self, force: bool = False):
+    def reindex_repos(self):
         """Reindex all repositories managed by the RepoManager."""
         logging.debug("Reindexing all repositories...")
 
         for repo in track(self.repo_manager.repos, description="Reindexing repos..."):
-            self.reindex_repo(repo, force=force)
+            self.reindex_repo(repo)
 
-    def _get_stages(self, name: str) -> list[dict[str, Any]]:
-        """Get all pipeline stages defined in the merged configuration.
+    def get_recipes(self) -> list[Repo]:
+        """Get all recipe repos available, sorted by priority (lower number first).
 
-        Returns:
-            List of stage definitions (dictionaries with 'name' and 'priority')
+        Recipes without a priority are placed at the end of the list.
         """
-        # 1. Get all pipeline definitions (the `[[stages]]` tables with name and priority).
-        pipeline_definitions = self.repo_manager.merged.getall(name)
-        flat_pipeline_steps = list(itertools.chain.from_iterable(pipeline_definitions))
+        recipes = [r for r in self.repo_manager.repos if r.kind() == "recipe"]
 
-        # 2. Sort the pipeline steps by their 'priority' field.
-        try:
-            sorted_pipeline = sorted(flat_pipeline_steps, key=lambda s: s["priority"])
-        except KeyError as e:
-            # Re-raise as a ValueError with a more descriptive message.
-            raise ValueError(
-                f"invalid stage definition: a stage is missing the required 'priority' key"
-            ) from e
-
-        logging.debug(
-            f"Found {len(sorted_pipeline)} pipeline steps to run in order of priority."
-        )
-        return sorted_pipeline
-
-    def run_all_stages(self):
-        """On the currently active session, run all processing stages
-
-        New design, not yet implemented:
-        * find all recipes
-        * for each target in the current selection:
-        *   select ONE recipe for processing that target (check recipe.auto.require.* conditions)
-        *   create a processing output directory (for high value final files)
-        *   create a temporary processing directory (for intermediate files - shared by all stages)
-        *   init session context (it will be shared for all following steps)
-        *   iterate over all light frame sessions in the current selection
-        *     for each session:
-        *       update context input and output files
-        *       run session.light stages
-        *   after all sessions are processed, run final.stack stages (using the shared context and temp dir)
+        # Sort recipes by priority (lower number first). If no priority specified,
+        # use float('inf') to push those to the end of the list.
+        def priority_key(r: Repo) -> float:
+            priority = r.get("recipe.priority")
+            return float(priority) if priority is not None else float("inf")
 
-        """
-        logging.info("--- Running all stages ---")
-
-        # 1. Get all pipeline definitions (the `[[stages]]` tables with name and priority).
-        sorted_pipeline = self._get_stages("stages")
-
-        self.init_context()
-        # 4. Iterate through the sorted pipeline and execute the associated tasks.
-        for step in sorted_pipeline:
-            step_name = step.get("name")
-            if not step_name:
-                raise ValueError("Invalid pipeline step found: missing 'name' key.")
-            self.run_pipeline_step(step_name)
-
-    def run_pipeline_step(self, step_name: str):
-        logging.info(f"--- Running pipeline step: '{step_name}' ---")
-
-        # 3. Get all available task definitions (the `[[stage]]` tables with tool, script, when).
-        task_definitions = self.repo_manager.merged.getall("stage")
-        all_tasks = list(itertools.chain.from_iterable(task_definitions))
-
-        # Find all tasks that should run during this pipeline step.
-        tasks_to_run = [task for task in all_tasks if task.get("when") == step_name]
-        for task in tasks_to_run:
-            self.run_stage(task)
-
-    def get_recipe_for_session(
-        self, session: SessionRow, step: dict[str, Any]
-    ) -> Repo | None:
+        recipes.sort(key=priority_key)
+
+        return recipes
+
+    def get_recipe_for_session(self, session: SessionRow, step: dict[str, Any]) -> Repo | None:
         """Try to find a recipe that can be used to process the given session for the given step name
         (master-dark, master-bias, light, stack, etc...)
 
@@ -757,7 +763,7 @@ class Starbash:
         and make the user pick (by throwing an exception?).
         """
         # Get all recipe repos - FIXME add a getall(kind) to RepoManager
-        recipe_repos = [r for r in self.repo_manager.repos if r.kind() == "recipe"]
+        recipe_repos = self.get_recipes()
 
         step_name = step.get("name")
         if not step_name:
@@ -767,13 +773,16 @@ class Starbash:
         if not input_name:
             raise ValueError("Invalid pipeline step found: missing 'input' key.")
 
-        imagetyp = session.get(get_column_name(Database.IMAGETYP_KEY))
+        # if input type is recipe we don't check for filetype match - because we'll just use files already in
+        # the tempdir
+        if input_name != "recipe":
+            imagetyp = session.get(get_column_name(Database.IMAGETYP_KEY))
 
-        if not imagetyp or input_name != self.aliases.normalize(imagetyp):
-            logging.debug(
-                f"Session imagetyp '{imagetyp}' does not match step input '{input_name}', skipping"
-            )
-            return None
+            if not imagetyp or input_name != self.aliases.normalize(imagetyp):
+                logging.debug(
+                    f"Session imagetyp '{imagetyp}' does not match step input '{input_name}', skipping"
+                )
+                return None
 
         # Get session metadata for checking requirements
         session_metadata = session.get("metadata", {})
@@ -782,21 +791,19 @@ class Starbash:
             # Check if this recipe has the requested stage
             stage_config = repo.get(f"recipe.stage.{step_name}")
             if not stage_config:
-                logging.debug(
-                    f"Recipe {repo.url} does not have stage '{step_name}', skipping"
-                )
+                logging.debug(f"Recipe {repo.url} does not have stage '{step_name}', skipping")
                 continue
 
             # Check auto.require conditions if they exist
 
             # If requirements are specified, check if session matches
-            required_filters = repo.get("auto.require.filter", [])
+            required_filters = repo.get("recipe.auto.require.filter", [])
             if required_filters:
                 session_filter = self.aliases.normalize(
-                    session_metadata.get(Database.FILTER_KEY)
+                    session_metadata.get(Database.FILTER_KEY), lenient=True
                 )
 
-                # Session must have a filter that matches one of the required filters
+                # Session must have AT LEAST one filter that matches one of the required filters
                 if not session_filter or session_filter not in required_filters:
                     logging.debug(
                         f"Recipe {repo.url} requires filters {required_filters}, "
@@ -804,10 +811,22 @@ class Starbash:
                     )
                     continue
 
-            required_cameras = repo.get("auto.require.camera", [])
+            required_color = repo.get("recipe.auto.require.color", False)
+            if required_color:
+                session_bayer = session_metadata.get("BAYERPAT")
+
+                # Session must be color (i.e. have a BAYERPAT header)
+                if not session_bayer:
+                    logging.debug(
+                        f"Recipe {repo.url} requires a color camera, "
+                        f"but session has no BAYERPAT header, skipping"
+                    )
+                    continue
+
+            required_cameras = repo.get("recipe.auto.require.camera", [])
             if required_cameras:
                 session_camera = self.aliases.normalize(
-                    session_metadata.get("INSTRUME")
+                    session_metadata.get("INSTRUME"), lenient=True
                 )  # Camera identifier
 
                 # Session must have a camera that matches one of the required cameras
@@ -825,323 +844,26 @@ class Starbash:
         # No matching recipe found
         return None
 
-    def run_master_stages(self):
-        """Generate any missing master frames
-
-        Steps:
-        * set all_tasks to be all tasks for when == "setup.master.bias"
-        * loop over all currently unfiltered sessions
-        * for each session loop across all_tasks
-        * if task input.type == the imagetyp for this current session
-        *    add_input_to_context() add the input files to the context (from the session)
-        *    run_stage(task) to generate the new master frame
-        """
-        sorted_pipeline = self._get_stages("master-stages")
-        sessions = self.search_session()
-        for session in track(sessions, description="Generating masters..."):
-            # 4. Iterate through the sorted pipeline and execute the associated tasks.
-            # FIXME unify the master vs normal step running code
-            for step in sorted_pipeline:
-                task = None
-                recipe = self.get_recipe_for_session(session, step)
-                if recipe:
-                    task = recipe.get("recipe.stage." + step["name"])
-
-                if task:
-                    input_config = task.get("input", {})
-                    input_type = input_config.get("type")
-                    if not input_type:
-                        raise ValueError(f"Task for step missing required input.type")
-
-                    # Create a default process dir in /tmp.
-                    # FIXME - eventually we should allow hashing or somesuch to keep reusing processing
-                    # dirs for particular targets?
-                    with ProcessingContext(self) as temp_dir:
-                        self.set_session_in_context(session)
-                        self.run_stage(task)
-
-    def init_context(self) -> None:
-        """Do common session init"""
-
-        # Context is preserved through all stages, so each stage can add new symbols to it for use by later stages
-        self.context = {}
-
-        # Update the context with runtime values.
-        runtime_context = {
-            # "masters": "/workspaces/starbash/images/masters",  # FIXME find this the correct way
-        }
-        self.context.update(runtime_context)
-
-    def set_session_in_context(self, session: SessionRow) -> None:
-        """adds to context from the indicated session:
-
-        Sets the following context variables based on the provided session:
-        * instrument - for the session
-        * date - the localtimezone date of the session
-        * imagetyp - the imagetyp of the session
-        * session - the current session row (joined with a typical image) (can be used to
-        find things like telescope, temperature ...)
-        * session_config - a short human readable description of the session - suitable for logs or filenames
-        """
-        # it is okay to give them the actual session row, because we're never using it again
-        self.context["session"] = session
-
-        instrument = session.get(get_column_name(Database.TELESCOP_KEY))
-        if instrument:
-            self.context["instrument"] = instrument
-
-        imagetyp = session.get(get_column_name(Database.IMAGETYP_KEY))
-        if imagetyp:
-            imagetyp = self.aliases.normalize(imagetyp)
-            self.context["imagetyp"] = imagetyp
-
-            # add a short human readable description of the session - suitable for logs or in filenames
-            session_config = f"{imagetyp}"
-
-            metadata = session.get("metadata", {})
-            filter = metadata.get(Database.FILTER_KEY)
-            if (imagetyp == "flat" or imagetyp == "light") and filter:
-                # we only care about filters in these cases
-                session_config += f"_{filter}"
-            if imagetyp == "dark":
-                exptime = session.get(get_column_name(Database.EXPTIME_KEY))
-                if exptime:
-                    session_config += f"_{int(float(exptime))}s"
-
-            self.context["session_config"] = session_config
-
-        date = session.get(get_column_name(Database.START_KEY))
-        if date:
-            self.context["date"] = to_shortdate(date)
-
-    def add_input_masters(self, stage: dict) -> None:
-        """based on input.masters add the correct master frames as context.master.<type> filepaths"""
-        session = self.context.get("session")
-        assert session is not None, "context.session should have been already set"
-
-        input_config = stage.get("input", {})
-        master_types: list[str] = input_config.get("masters", [])
-        for master_type in master_types:
-            masters = self.get_master_images(
-                imagetyp=master_type, reference_session=session
-            )
-            if not masters:
-                raise RuntimeError(
-                    f"No master frames of type '{master_type}' found for stage '{stage.get('name')}'"
-                )
-
-            context_master = self.context.setdefault("master", {})
-
-            if len(masters) > 1:
-                logging.debug(
-                    f"Multiple ({len(masters)}) master frames of type '{master_type}' found, using first. FIXME."
-                )
-
-            # Try to rank the images by desirability
-            masters = self.score_candidates(masters, session)
-
-            self._add_image_abspath(masters[0])  # make sure abspath is populated
-            selected_master = masters[0]["abspath"]
-            logging.info(f"For master '{master_type}', using: {selected_master}")
-
-            context_master[master_type] = selected_master
-
-    def add_input_files(self, stage: dict) -> None:
-        """adds to context.input_files based on the stage input config"""
-        input_config = stage.get("input")
-        input_required = 0
-        if input_config:
-            # if there is an "input" dict, we assume input.required is true if unset
-            input_required = input_config.get("required", 0)
-            source = input_config.get("source")
-            if source is None:
-                raise ValueError(
-                    f"Stage '{stage.get('name')}' has invalid 'input' configuration: missing 'source'"
-                )
-            if source == "path":
-                # The path might contain context variables that need to be expanded.
-                # path_pattern = expand_context(input_config["path"], context)
-                path_pattern = input_config["path"]
-                input_files = glob.glob(path_pattern, recursive=True)
-
-                self.context["input_files"] = (
-                    input_files  # Pass in the file list via the context dict
-                )
-            elif source == "repo":
-                # Get images for this session (by pulling from repo)
-                session = self.context.get("session")
-                assert (
-                    session is not None
-                ), "context.session should have been already set"
-
-                images = self.get_session_images(session)
-                logging.debug(f"Using {len(images)} files as input_files")
-                self.context["input_files"] = [
-                    img["abspath"] for img in images
-                ]  # Pass in the file list via the context dict
-            else:
-                raise ValueError(
-                    f"Stage '{stage.get('name')}' has invalid 'input' source: {source}"
-                )
-
-            # FIXME compare context.output to see if it already exists and is newer than the input files, if so skip processing
-        else:
-            # The script doesn't mention input, therefore assume it doesn't want input_files
-            if "input_files" in self.context:
-                del self.context["input_files"]
-
-        if input_required and len(self.context.get("input_files", [])) < input_required:
-            raise RuntimeError(f"Stage requires at least {input_required} input files")
-
-    def add_output_path(self, stage: dict) -> None:
-        """Adds output path information to context based on the stage output config.
-
-        Sets the following context variables:
-        - context.output.root_path - base path of the destination repo
-        - context.output.base_path - full path without file extension
-        - context.output.suffix - file extension (e.g., .fits or .fit.gz)
-        - context.output.full_path - complete output file path
-        - context.output.repo - the destination Repo (if applicable)
-        """
-        output_config = stage.get("output")
-        if not output_config:
-            # No output configuration, remove any existing output from context
-            if "output" in self.context:
-                del self.context["output"]
-            return
-
-        dest = output_config.get("dest")
-        if not dest:
-            raise ValueError(
-                f"Stage '{stage.get('description', 'unknown')}' has 'output' config but missing 'dest'"
-            )
-
-        if dest == "repo":
-            # Find the destination repo by type/kind
-            output_type = output_config.get("type")
-            if not output_type:
-                raise ValueError(
-                    f"Stage '{stage.get('description', 'unknown')}' has output.dest='repo' but missing 'type'"
-                )
-
-            # Find the repo with matching kind
-            dest_repo = self.repo_manager.get_repo_by_kind(output_type)
-            if not dest_repo:
-                raise ValueError(
-                    f"No repository found with kind '{output_type}' for output destination"
-                )
-
-            repo_base = dest_repo.get_path()
-            if not repo_base:
-                raise ValueError(f"Repository '{dest_repo.url}' has no filesystem path")
-
-            repo_relative: str | None = dest_repo.get("repo.relative")
-            if not repo_relative:
-                raise ValueError(
-                    f"Repository '{dest_repo.url}' is missing 'repo.relative' configuration"
-                )
-
-            # we support context variables in the relative path
-            repo_relative = expand_context_unsafe(repo_relative, self.context)
-            full_path = repo_base / repo_relative
-
-            # base_path but without spaces - because Siril doesn't like that
-            full_path = Path(str(full_path).replace(" ", r"_"))
-
-            base_path = full_path.parent / full_path.stem
-
-            # Set context variables as documented in the TOML
-            self.context["output"] = {
-                # "root_path": repo_relative, not needed I think
-                "base_path": base_path,
-                # "suffix": full_path.suffix, not needed I think
-                "full_path": full_path,
-                "repo": dest_repo,
-            }
-        else:
-            raise ValueError(
-                f"Unsupported output destination type: {dest}. Only 'repo' is currently supported."
-            )
-
-    def run_stage(self, stage: dict) -> None:
-        """
-        Executes a single processing stage.
-
-        Args:
-            stage: A dictionary representing the stage configuration, containing
-                   at least 'tool' and 'script' keys.
-        """
-        stage_desc = stage.get("description", "(missing description)")
-        stage_disabled = stage.get("disabled", False)
-        if stage_disabled:
-            logging.info(f"Skipping disabled stage: {stage_desc}")
-            return
-
-        logging.info(f"Running stage: {stage_desc}")
-
-        tool_dict = stage.get("tool")
-        if not tool_dict:
-            raise ValueError(
-                f"Stage '{stage.get('name')}' is missing a 'tool' definition."
-            )
-        tool_name = tool_dict.get("name")
-        if not tool_name:
-            raise ValueError(
-                f"Stage '{stage.get('name')}' is missing a 'tool.name' definition."
-            )
-        tool = tools.get(tool_name)
-        if not tool:
-            raise ValueError(
-                f"Tool '{tool_name}' for stage '{stage.get('name')}' not found."
-            )
-        logging.debug(f"Using tool: {tool_name}")
-        tool.set_defaults()
-
-        # Allow stage to override tool timeout if specified
-        tool_timeout = tool_dict.get("timeout")
-        if tool_timeout is not None:
-            tool.timeout = float(tool_timeout)
-            logging.debug(f"Using tool timeout: {tool.timeout} seconds")
-
-        script_filename = stage.get("script-file", tool.default_script_file)
-        if script_filename:
-            source = stage.source  # type: ignore (was monkeypatched by repo)
-            script = source.read(script_filename)
-        else:
-            script = stage.get("script")
-
-        if script is None:
-            raise ValueError(
-                f"Stage '{stage.get('name')}' is missing a 'script' or 'script-file' definition."
-            )
-
-        # This allows recipe TOML to define their own default variables.
-        # (apply all of the changes to context that the task demands)
-        stage_context = stage.get("context", {})
-        self.context.update(stage_context)
-        self.add_input_files(stage)
-        self.add_input_masters(stage)
-        self.add_output_path(stage)
-
-        # if the output path already exists and is newer than all input files, skip processing
-        output_info: dict | None = self.context.get("output")
-        if output_info:
-            output_path = output_info.get("full_path")
-
-            if output_path and os.path.exists(output_path):
-                logging.info(
-                    f"Output file already exists, skipping processing: {output_path}"
-                )
-                return
-
-        tool.run_in_temp_dir(script, context=self.context)
-
-        # verify context.output was created if it was specified
-        output_info: dict | None = self.context.get("output")
-        if output_info:
-            output_path = output_info.get("full_path")
+    def filter_sessions_with_lights(self, sessions: list[SessionRow]) -> list[SessionRow]:
+        """Filter sessions to only those that contain light frames."""
+        filtered_sessions: list[SessionRow] = []
+        for s in sessions:
+            imagetyp_val = s.get(get_column_name(Database.IMAGETYP_KEY))
+            if imagetyp_val is None:
+                continue
+            if self.aliases.normalize(str(imagetyp_val)) == "light":
+                filtered_sessions.append(s)
+        return filtered_sessions
 
-            if not output_path or not os.path.exists(output_path):
-                raise RuntimeError(f"Expected output file not found: {output_path}")
-            else:
-                self.add_image_to_db(output_info["repo"], Path(output_path), force=True)
+    def filter_sessions_by_target(
+        self, sessions: list[SessionRow], target: str
+    ) -> list[SessionRow]:
+        """Filter sessions to only those that match the given target name."""
+        filtered_sessions: list[SessionRow] = []
+        for s in sessions:
+            obj_val = s.get(get_column_name(Database.OBJECT_KEY))
+            if obj_val is None:
+                continue
+            if normalize_target_name(str(obj_val)) == target:
+                filtered_sessions.append(s)
+        return filtered_sessions