starbash 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl

starbash/app.py

@@ -1,6 +1,8 @@
 import logging
 from importlib import resources
+import os
 from pathlib import Path
+import tempfile
 import typer
 import tomlkit
 from tomlkit.toml_file import TOMLFile
@@ -11,16 +13,20 @@ import itertools
 from rich.progress import track
 from rich.logging import RichHandler
 import shutil
+from datetime import datetime
+import rich.console
+import copy
 
 import starbash
-from starbash import console, _is_test_env
-from starbash.database import Database
-from repo.manager import Repo
-from starbash.tool import Tool
+from starbash import console, _is_test_env, to_shortdate
+from starbash.database import Database, SessionRow, ImageRow, get_column_name
+from repo import Repo, repo_suffix
+from starbash.toml import toml_from_template
+from starbash.tool import Tool, expand_context, expand_context_unsafe
 from repo import RepoManager
 from starbash.tool import tools
 from starbash.paths import get_user_config_dir, get_user_data_dir
-from starbash.selection import Selection
+from starbash.selection import Selection, where_tuple
 from starbash.analytics import (
     NopAnalytics,
     analytics_exception,
@@ -29,12 +35,17 @@ from starbash.analytics import (
     analytics_start_transaction,
 )
 
+# Type aliases for better documentation
 
-def setup_logging():
+
+def setup_logging(stderr: bool = False):
     """
     Configures basic logging.
     """
-    handlers = [RichHandler(rich_tracebacks=True)] if not _is_test_env else []
+    console = rich.console.Console(stderr=stderr)
+    handlers = (
+        [RichHandler(console=console, rich_tracebacks=True)] if not _is_test_env else []
+    )
     logging.basicConfig(
         level=starbash.log_filter_level,  # use the global log filter level
         format="%(message)s",
@@ -46,25 +57,19 @@ def setup_logging():
 def get_user_config_path() -> Path:
     """Returns the path to the user config file."""
     config_dir = get_user_config_dir()
-    return config_dir / "starbash.toml"
+    return config_dir / repo_suffix
 
 
 def create_user() -> Path:
     """Create user directories if they don't exist yet."""
     path = get_user_config_path()
     if not path.exists():
-        tomlstr = (
-            resources.files("starbash")
-            .joinpath("templates/userconfig.toml")
-            .read_text()
-        )
-        toml = tomlkit.parse(tomlstr)
-        TOMLFile(path).write(toml)
+        toml_from_template("userconfig", path)
         logging.info(f"Created user config file: {path}")
     return get_user_config_dir()
 
 
-def copy_images_to_dir(images: list[dict[str, Any]], output_dir: Path) -> None:
+def copy_images_to_dir(images: list[ImageRow], output_dir: Path) -> None:
     """Copy images to the specified output directory (using symbolic links if possible)."""
 
     # Export images
@@ -113,15 +118,23 @@ def copy_images_to_dir(images: list[dict[str, Any]], output_dir: Path) -> None:
         console.print(f"  [red]Errors: {error_count} files[/red]")
 
 
+def imagetyp_equals(imagetyp1: str, imagetyp2: str) -> bool:
+    """Imagetyps (BIAS, Dark, FLAT, flats) have a number of slightly different convetions.
+    Do a sloppy equality check.
+
+    Eventually handle non english variants by using the repos aliases table."""
+    return imagetyp1.strip().lower() == imagetyp2.strip().lower()
+
+
 class Starbash:
     """The main Starbash application class."""
 
-    def __init__(self, cmd: str = "unspecified"):
+    def __init__(self, cmd: str = "unspecified", stderr_logging: bool = False):
         """
         Initializes the Starbash application by loading configurations
         and setting up the repository manager.
         """
-        setup_logging()
+        setup_logging(stderr=stderr_logging)
         logging.info("Starbash starting...")
 
         # Load app defaults and initialize the repository manager
@@ -149,23 +162,45 @@ class Starbash:
         )
         # self.repo_manager.dump()
 
-        self.db = Database()
+        self._db = None  # Lazy initialization - only create when accessed
         self.session_query = None  # None means search all sessions
 
-        # Initialize selection state
-        data_dir = get_user_data_dir()
-        selection_file = data_dir / "selection.json"
-        self.selection = Selection(selection_file)
+        # Initialize selection state (stored in user config repo)
+        self.selection = Selection(self.user_repo)
 
         # FIXME, call reindex somewhere and also index whenever new repos are added
         # self.reindex_repos()
 
+    @property
+    def db(self) -> Database:
+        """Lazy initialization of database - only created as needed."""
+        if self._db is None:
+            self._db = Database()
+            # Ensure all repos are registered in the database
+            self.repo_db_update()
+        return self._db
+
+    def repo_db_update(self) -> None:
+        """Update the database with all managed repositories.
+
+        Iterates over all repos in the RepoManager and ensures each one
+        has a record in the repos table. This is called during lazy database
+        initialization to prepare repo_id values for image insertion.
+        """
+        if self._db is None:
+            return
+
+        for repo in self.repo_manager.repos:
+            self._db.upsert_repo(repo.url)
+            logging.debug(f"Registered repo in database: {repo.url}")
+
     # --- Lifecycle ---
     def close(self) -> None:
         self.analytics.__exit__(None, None, None)
 
         analytics_shutdown()
-        self.db.close()
+        if self._db is not None:
+            self._db.close()
 
     # Context manager support
     def __enter__(self) -> "Starbash":
@@ -180,6 +215,7 @@ class Starbash:
         return handled
 
     def _add_session(self, f: str, image_doc_id: int, header: dict) -> None:
+        """We just added a new image, create or update its session entry as needed."""
         filter = header.get(Database.FILTER_KEY, "unspecified")
         image_type = header.get(Database.IMAGETYP_KEY)
         date = header.get(Database.DATE_OBS_KEY)
@@ -205,13 +241,154 @@ class Starbash:
             session = self.db.get_session(new)
             self.db.upsert_session(new, existing=session)
 
-    def search_session(self) -> list[dict[str, Any]]:
+    def guess_sessions(
+        self, ref_session: SessionRow, want_type: str
+    ) -> list[SessionRow]:
+        """Given a particular session type (i.e. FLAT or BIAS etc...) and an
+        existing session (which is assumed to generally be a LIGHT frame based session):
+
+        Return a list of possible sessions which would be acceptable.  The more desirable
+        matches are first in the list.  Possibly in the future I might have a 'score' and reason
+        given for each ranking.
+
+        The following critera MUST match to be acceptable:
+        * matches requested imagetyp.
+        * same filter as reference session (in the case want_type==FLAT only)
+        * same telescope as reference session
+
+        Quality is determined by (most important first):
+        * temperature of CCD-TEMP is closer to the reference session
+        * smaller DATE-OBS delta to the reference session
+
+        Eventually the code will check the following for 'nice to have' (but not now):
+        * TBD
+
+        Possibly eventually this code could be moved into recipes.
+
+        """
+        # Get reference image to access CCD-TEMP and DATE-OBS
+        metadata: dict = ref_session.get("metadata", {})
+        ref_temp = metadata.get("CCD-TEMP", None)
+        ref_date_str = metadata.get(Database.DATE_OBS_KEY)
+
+        # Parse reference date for time delta calculations
+        ref_date = None
+        if ref_date_str:
+            try:
+                ref_date = datetime.fromisoformat(ref_date_str)
+            except (ValueError, TypeError):
+                logging.warning(f"Malformed session ref date: {ref_date_str}")
+
+        # Build search conditions - MUST match criteria
+        conditions = {
+            Database.IMAGETYP_KEY: want_type,
+            Database.TELESCOP_KEY: ref_session[get_column_name(Database.TELESCOP_KEY)],
+        }
+
+        # For FLAT frames, filter must match the reference session
+        if want_type.upper() == "FLAT":
+            conditions[Database.FILTER_KEY] = ref_session[
+                get_column_name(Database.FILTER_KEY)
+            ]
+
+        # Search for candidate sessions
+        candidates = self.db.search_session(where_tuple(conditions))
+
+        # Now score and sort the candidates
+        scored_candidates = []
+
+        for candidate in candidates:
+            score = 0.0
+
+            # Get candidate image metadata to access CCD-TEMP and DATE-OBS
+            try:
+                candidate_image = candidate.get("metadata", {})
+
+                # Score by CCD-TEMP difference (most important)
+                # Lower temperature difference = better score
+                if ref_temp is not None:
+                    candidate_temp = candidate_image.get("CCD-TEMP")
+                    if candidate_temp is not None:
+                        try:
+                            temp_diff = abs(float(ref_temp) - float(candidate_temp))
+                            # Use exponential decay: closer temps get much better scores
+                            # Perfect match (0°C diff) = 1000, 1°C diff ≈ 368, 2°C diff ≈ 135
+                            score += 1000 * (2.718 ** (-temp_diff))
+                        except (ValueError, TypeError):
+                            # If we can't parse temps, give a neutral score
+                            score += 0
+
+                # Score by date/time proximity (secondary importance)
+                if ref_date is not None:
+                    candidate_date_str = candidate_image.get(Database.DATE_OBS_KEY)
+                    if candidate_date_str:
+                        try:
+                            candidate_date = datetime.fromisoformat(candidate_date_str)
+                            time_delta = abs(
+                                (ref_date - candidate_date).total_seconds()
+                            )
+                            # Closer in time = better score
+                            # Same day ≈ 100, 7 days ≈ 37, 30 days ≈ 9
+                            # Using 7-day half-life
+                            score += 100 * (2.718 ** (-time_delta / (7 * 86400)))
+                        except (ValueError, TypeError):
+                            logging.warning(
+                                f"Could not parse candidate date: {candidate_date_str}"
+                            )
+
+                scored_candidates.append((score, candidate))
+
+            except (AssertionError, KeyError) as e:
+                # If we can't get the session image, log and skip this candidate
+                logging.warning(
+                    f"Could not score candidate session {candidate.get('id')}: {e}"
+                )
+                continue
+
+        # Sort by score (highest first) and return just the sessions
+        scored_candidates.sort(key=lambda x: x[0], reverse=True)
+
+        return [candidate for score, candidate in scored_candidates]
+
+    def search_session(self) -> list[SessionRow]:
         """Search for sessions, optionally filtered by the current selection."""
         # Get query conditions from selection
         conditions = self.selection.get_query_conditions()
         return self.db.search_session(conditions)
 
-    def get_session_images(self, session_id: int) -> list[dict[str, Any]]:
+    def _reconstruct_image_path(self, image: ImageRow) -> ImageRow:
+        """Reconstruct absolute path from image row containing repo_url and relative path.
+
+        Args:
+            image: Image record with 'repo_url' and 'path' (relative) fields
+
+        Returns:
+            Modified image record with 'path' as absolute path
+        """
+        repo_url = image.get("repo_url")
+        relative_path = image.get("path")
+
+        if repo_url and relative_path:
+            repo = self.repo_manager.get_repo_by_url(repo_url)
+            if repo:
+                absolute_path = repo.resolve_path(relative_path)
+                image["path"] = str(absolute_path)
+
+        return image
+
+    def get_session_image(self, session: SessionRow) -> ImageRow:
+        """
+        Get the reference ImageRow for a session with absolute path.
+        """
+        images = self.db.search_image(
+            {Database.ID_KEY: session[get_column_name(Database.IMAGE_DOC_KEY)]}
+        )
+        assert (
+            len(images) == 1
+        ), f"Expected exactly one reference for session, found {len(images)}"
+        return self._reconstruct_image_path(images[0])
+
+    def get_session_images(self, session: SessionRow) -> list[ImageRow]:
         """
         Get all images belonging to a specific session.
 
@@ -229,24 +406,20 @@ class Starbash:
         Raises:
             ValueError: If session_id is not found in the database
         """
-        # First get the session details
-        session = self.db.get_session_by_id(session_id)
-        if session is None:
-            raise ValueError(f"Session with id {session_id} not found")
-
         # Query images that match ALL session criteria including date range
         conditions = {
-            Database.FILTER_KEY: session[Database.FILTER_KEY],
-            Database.IMAGETYP_KEY: session[Database.IMAGETYP_KEY],
-            Database.OBJECT_KEY: session[Database.OBJECT_KEY],
-            Database.TELESCOP_KEY: session[Database.TELESCOP_KEY],
-            "date_start": session[Database.START_KEY],
-            "date_end": session[Database.END_KEY],
+            Database.FILTER_KEY: session[get_column_name(Database.FILTER_KEY)],
+            Database.IMAGETYP_KEY: session[get_column_name(Database.IMAGETYP_KEY)],
+            Database.OBJECT_KEY: session[get_column_name(Database.OBJECT_KEY)],
+            Database.TELESCOP_KEY: session[get_column_name(Database.TELESCOP_KEY)],
+            "date_start": session[get_column_name(Database.START_KEY)],
+            "date_end": session[get_column_name(Database.END_KEY)],
         }
 
         # Single query with all conditions
         images = self.db.search_image(conditions)
-        return images if images else []
+        # Reconstruct absolute paths for all images
+        return [self._reconstruct_image_path(img) for img in images] if images else []
 
     def remove_repo_ref(self, url: str) -> None:
         """
@@ -258,6 +431,8 @@ class Starbash:
         Raises:
             ValueError: If the repository URL is not found in user configuration
         """
+        self.db.remove_repo(url)
+
         # Get the repo-ref list from user config
         repo_refs = self.user_repo.config.get("repo-ref")
 
@@ -283,6 +458,10 @@ class Starbash:
 
     def reindex_repo(self, repo: Repo, force: bool = False):
         """Reindex all repositories managed by the RepoManager."""
+
+        # make sure this new repo is listed in the repos table
+        self.repo_db_update()  # not really ideal, a more optimal version would just add the new repo
+
         # FIXME, add a method to get just the repos that contain images
         if repo.is_scheme("file") and repo.kind != "recipe":
             logging.debug("Reindexing %s...", repo.url)
@@ -303,7 +482,10 @@ class Starbash:
             ):
                 # progress.console.print(f"Indexing {f}...")
                 try:
-                    found = self.db.get_image(str(f))
+                    # Convert absolute path to relative path within repo
+                    relative_path = f.relative_to(path)
+
+                    found = self.db.get_image(repo.url, str(relative_path))
                     if not found or force:
                         # Read and log the primary header (HDU 0)
                         with fits.open(str(f), memmap=False) as hdul:
@@ -319,8 +501,9 @@ class Starbash:
                                 if (not whitelist) or (key in whitelist):
                                     headers[key] = value
                             logging.debug("Headers for %s: %s", f, headers)
-                            headers["path"] = str(f)
-                            image_doc_id = self.db.upsert_image(headers)
+                            # Store relative path in database
+                            headers["path"] = str(relative_path)
+                            image_doc_id = self.db.upsert_image(headers, repo.url)
 
                             if not found:
                                 # Update the session infos, but ONLY on first file scan
@@ -337,10 +520,6 @@ class Starbash:
         for repo in track(self.repo_manager.repos, description="Reindexing repos..."):
             self.reindex_repo(repo, force=force)
 
-    def test_processing(self):
-        """A crude test of image processing pipeline - FIXME move into testing"""
-        self.run_all_stages()
-
     def run_all_stages(self):
         """On the currently active session, run all processing stages"""
         logging.info("--- Running all stages ---")
@@ -358,30 +537,77 @@ class Starbash:
                 f"invalid stage definition: a stage is missing the required 'priority' key"
             ) from e
 
-        # 3. Get all available task definitions (the `[[stage]]` tables with tool, script, when).
-        task_definitions = self.repo_manager.merged.getall("stage")
-        all_tasks = list(itertools.chain.from_iterable(task_definitions))
-
         logging.info(
             f"Found {len(sorted_pipeline)} pipeline steps to run in order of priority."
         )
 
-        self.start_session()
+        self.init_context()
         # 4. Iterate through the sorted pipeline and execute the associated tasks.
         for step in sorted_pipeline:
             step_name = step.get("name")
             if not step_name:
                 raise ValueError("Invalid pipeline step found: missing 'name' key.")
+            self.run_pipeline_step(step_name)
+
+    def run_pipeline_step(self, step_name: str):
+        logging.info(f"--- Running pipeline step: '{step_name}' ---")
 
-            logging.info(
-                f"--- Running pipeline step: '{step_name}' (Priority: {step['priority']}) ---"
+        # 3. Get all available task definitions (the `[[stage]]` tables with tool, script, when).
+        task_definitions = self.repo_manager.merged.getall("stage")
+        all_tasks = list(itertools.chain.from_iterable(task_definitions))
+
+        # Find all tasks that should run during this pipeline step.
+        tasks_to_run = [task for task in all_tasks if task.get("when") == step_name]
+        for task in tasks_to_run:
+            self.run_stage(task)
+
+    def run_master_stages(self):
+        """Generate any missing master frames
+
+        Steps:
+        * set all_tasks to be all tasks for when == "setup.masters"
+        * loop over all currently unfiltered sessions
+        * for each session loop across all_tasks
+        * if task input.type == the imagetyp for this current session
+        *    add_input_to_context() add the input files to the context (from the session)
+        *    run_stage(task) to generate the new master frame
+        """
+        sessions = self.search_session()
+        for session in sessions:
+            imagetyp = session[get_column_name(Database.IMAGETYP_KEY)]
+            logging.debug(
+                f"Processing session ID {session[get_column_name(Database.ID_KEY)]} with imagetyp '{imagetyp}'"
             )
-            # Find all tasks that should run during this pipeline step.
-            tasks_to_run = [task for task in all_tasks if task.get("when") == step_name]
-            for task in tasks_to_run:
-                self.run_stage(task)
 
-    def start_session(self) -> None:
+            # 3. Get all available task definitions (the `[[stage]]` tables with tool, script, when).
+            task_definitions = self.repo_manager.merged.getall("stage")
+            all_tasks = list(itertools.chain.from_iterable(task_definitions))
+
+            # Find all tasks that should run during the "setup.masters" step.
+            tasks_to_run = [
+                task for task in all_tasks if task.get("when") == "setup.masters"
+            ]
+
+            for task in tasks_to_run:
+                input_config = task.get("input", {})
+                input_type = input_config.get("type")
+                if imagetyp_equals(input_type, imagetyp):
+                    logging.info(
+                        f"  Running master stage task for imagetyp '{imagetyp}'"
+                    )
+
+                    # Create a default process dir in /tmp, though more advanced 'session' based workflows will
+                    # probably override this and place it somewhere persistent.
+                    with tempfile.TemporaryDirectory(prefix="session_tmp_") as temp_dir:
+                        logging.debug(
+                            f"Created temporary session directory: {temp_dir}"
+                        )
+                        self.init_context()
+                        self.context["process_dir"] = temp_dir
+                        self.add_session_to_context(session)
+                        self.run_stage(task)
+
+    def init_context(self) -> None:
         """Do common session init"""
 
         # Context is preserved through all stages, so each stage can add new symbols to it for use by later stages
@@ -389,11 +615,152 @@ class Starbash:
 
         # Update the context with runtime values.
         runtime_context = {
-            "process_dir": "/workspaces/starbash/images/process",  # FIXME - create/find this more correctly per session
             "masters": "/workspaces/starbash/images/masters",  # FIXME find this the correct way
         }
         self.context.update(runtime_context)
 
+    def add_session_to_context(self, session: SessionRow) -> None:
+        """adds to context from the indicated session:
+        * input_files - all of the files mentioned in the session
+        * instrument - for the session
+        * date - the localtimezone date of the session
+        * imagetyp - the imagetyp of the session
+        * session - the current session row (joined with a typical image) (can be used to
+        find things like telescope, temperature ...)
+        """
+        # Get images for this session
+        images = self.get_session_images(session)
+        logging.debug(f"Adding {len(images)} files as context.input_files")
+        self.context["input_files"] = [
+            img["path"] for img in images
+        ]  # Pass in the file list via the context dict
+
+        # it is okay to give them the actual session row, because we're never using it again
+        self.context["session"] = session
+
+        instrument = session.get(get_column_name(Database.TELESCOP_KEY))
+        if instrument:
+            self.context["instrument"] = instrument
+
+        imagetyp = session.get(get_column_name(Database.IMAGETYP_KEY))
+        if imagetyp:
+            self.context["imagetyp"] = imagetyp
+
+        date = session.get(get_column_name(Database.START_KEY))
+        if date:
+            self.context["date"] = to_shortdate(date)
+
+    def add_input_files(self, stage: dict) -> None:
+        """adds to context.input_files based on the stage input config"""
+        input_config = stage.get("input")
+        input_required = False
+        if input_config:
+            # if there is an "input" dict, we assume input.required is true if unset
+            input_required = input_config.get("required", True)
+            source = input_config.get("source")
+            if source is None:
+                raise ValueError(
+                    f"Stage '{stage.get('name')}' has invalid 'input' configuration: missing 'source'"
+                )
+            if source == "path":
+                # The path might contain context variables that need to be expanded.
+                # path_pattern = expand_context(input_config["path"], context)
+                path_pattern = input_config["path"]
+                input_files = glob.glob(path_pattern, recursive=True)
+
+                self.context["input_files"] = (
+                    input_files  # Pass in the file list via the context dict
+                )
+            elif source == "repo":
+                # We expect that higher level code has already added the correct input files
+                # to the context
+                if not "input_files" in self.context:
+                    raise RuntimeError(
+                        "Input config specifies 'repo' but no 'input_files' found in context"
+                    )
+            else:
+                raise ValueError(
+                    f"Stage '{stage.get('name')}' has invalid 'input' source: {source}"
+                )
+
+            # FIXME compare context.output to see if it already exists and is newer than the input files, if so skip processing
+        else:
+            # The script doesn't mention input, therefore assume it doesn't want input_files
+            if "input_files" in self.context:
+                del self.context["input_files"]
+
+        if input_required and not "input_files" in self.context:
+            raise RuntimeError("No input files found for stage")
+
+    def add_output_path(self, stage: dict) -> None:
+        """Adds output path information to context based on the stage output config.
+
+        Sets the following context variables:
+        - context.output.root_path - base path of the destination repo
+        - context.output.base_path - full path without file extension
+        - context.output.suffix - file extension (e.g., .fits or .fit.gz)
+        - context.output.full_path - complete output file path
+        """
+        output_config = stage.get("output")
+        if not output_config:
+            # No output configuration, remove any existing output from context
+            if "output" in self.context:
+                del self.context["output"]
+            return
+
+        dest = output_config.get("dest")
+        if not dest:
+            raise ValueError(
+                f"Stage '{stage.get('description', 'unknown')}' has 'output' config but missing 'dest'"
+            )
+
+        if dest == "repo":
+            # Find the destination repo by type/kind
+            output_type = output_config.get("type")
+            if not output_type:
+                raise ValueError(
+                    f"Stage '{stage.get('description', 'unknown')}' has output.dest='repo' but missing 'type'"
+                )
+
+            # Find the repo with matching kind
+            dest_repo = self.repo_manager.get_repo_by_kind(output_type)
+            if not dest_repo:
+                raise ValueError(
+                    f"No repository found with kind '{output_type}' for output destination"
+                )
+
+            repo_base = dest_repo.get_path()
+            if not repo_base:
+                raise ValueError(f"Repository '{dest_repo.url}' has no filesystem path")
+
+            repo_relative: str | None = dest_repo.get("repo.relative")
+            if not repo_relative:
+                raise ValueError(
+                    f"Repository '{dest_repo.url}' is missing 'repo.relative' configuration"
+                )
+
+            # we support context variables in the relative path
+            repo_relative = expand_context_unsafe(repo_relative, self.context)
+            full_path = repo_base / repo_relative
+
+            # base_path but without spaces - because Siril doesn't like that
+            full_path = Path(str(full_path).replace(" ", r"_"))
+
+            base_path = full_path.parent / full_path.stem
+
+            # Set context variables as documented in the TOML
+            self.context["output"] = {
+                # "root_path": repo_relative, not needed I think
+                "base_path": base_path,
+                # "suffix": full_path.suffix, not needed I think
+                "full_path": full_path,
+            }
+
+        else:
+            raise ValueError(
+                f"Unsupported output destination type: {dest}. Only 'repo' is currently supported."
+            )
+
     def run_stage(self, stage: dict) -> None:
         """
         Executes a single processing stage.
@@ -435,30 +802,29 @@ class Starbash:
             )
 
         # This allows recipe TOML to define their own default variables.
+        # (apply all of the changes to context that the task demands)
         stage_context = stage.get("context", {})
         self.context.update(stage_context)
+        self.add_input_files(stage)
+        self.add_output_path(stage)
 
-        # Assume no files for this stage
-        if "input_files" in self.context:
-            del self.context["input_files"]
+        # if the output path already exists and is newer than all input files, skip processing
+        output_info: dict | None = self.context.get("output")
+        if output_info:
+            output_path = output_info.get("full_path")
 
-        input_files = []
-        input_config = stage.get("input")
-        input_required = False
-        if input_config:
-            # if there is an "input" dict, we assume input.required is true if unset
-            input_required = input_config.get("required", True)
-            if "path" in input_config:
-                # The path might contain context variables that need to be expanded.
-                # path_pattern = expand_context(input_config["path"], context)
-                path_pattern = input_config["path"]
-                input_files = glob.glob(path_pattern, recursive=True)
+            if output_path and os.path.exists(output_path):
+                logging.info(
+                    f"Output file already exists, skipping processing: {output_path}"
+                )
+                return
 
-            self.context["input_files"] = (
-                input_files  # Pass in the file list via the context dict
-            )
+        tool.run_in_temp_dir(script, context=self.context)
 
-        if input_required and not input_files:
-            raise RuntimeError("No input files found for stage")
+        # verify context.output was created if it was specified
+        output_info: dict | None = self.context.get("output")
+        if output_info:
+            output_path = output_info.get("full_path")
 
-        tool.run_in_temp_dir(script, context=self.context)
+            if not output_path or not os.path.exists(output_path):
+                raise RuntimeError(f"Expected output file not found: {output_path}")