starbash 0.1.10__tar.gz → 0.1.11__tar.gz

{starbash-0.1.10 → starbash-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starbash
-Version: 0.1.10
+Version: 0.1.11
 Summary: A tool for automating/standardizing/sharing astrophotography workflows.
 License-File: LICENSE
 Author: Kevin Hester

{starbash-0.1.10 → starbash-0.1.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "starbash"
-version = "0.1.10"
+version = "0.1.11"
 description = "A tool for automating/standardizing/sharing astrophotography workflows."
 authors = ["Kevin Hester <kevinh@geeksville.com>"]
 readme = "README.md"

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/app.py

@@ -22,7 +22,7 @@ import starbash
 from starbash import console, _is_test_env, to_shortdate
 from starbash.aliases import Aliases
 from starbash.database import Database, SessionRow, ImageRow, get_column_name
-from repo import Repo, repo_suffix
+from repo import Repo, repo, repo_suffix
 from starbash.toml import toml_from_template
 from starbash.tool import Tool, expand_context, expand_context_unsafe
 from repo import RepoManager
@@ -124,6 +124,33 @@ def copy_images_to_dir(images: list[ImageRow], output_dir: Path) -> None:
         console.print(f"  [red]Errors: {error_count} files[/red]")
 
 
+class ProcessingContext(tempfile.TemporaryDirectory):
+    """For processing a set of sessions for a particular target.
+
+    Keeps a shared temporary directory for intermediate files.  We expose the path to that
+    directory in context["process_dir"].
+    """
+
+    def __init__(self, starbash: "Starbash"):
+        super().__init__(prefix="sbprocessing_")
+        self.sb = starbash
+        logging.debug(f"Created processing context at {self.name}")
+
+        self.sb.init_context()
+        self.sb.context["process_dir"] = self.name
+
+    def __enter__(self) -> "ProcessingContext":
+        return super().__enter__()
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        logging.debug(f"Cleaning up processing context at {self.name}")
+
+        # unregister our process dir
+        self.sb.context.pop("process_dir", None)
+
+        super().__exit__(exc_type, exc_value, traceback)
+
+
 class Starbash:
     """The main Starbash application class."""
 
@@ -674,7 +701,22 @@ class Starbash:
         return sorted_pipeline
 
     def run_all_stages(self):
-        """On the currently active session, run all processing stages"""
+        """On the currently active session, run all processing stages
+
+        New design, not yet implemented:
+        * find all recipes
+        * for each target in the current selection:
+        *   select ONE recipe for processing that target (check recipe.auto.require.* conditions)
+        *   create a processing output directory (for high value final files)
+        *   create a temporary processing directory (for intermediate files - shared by all stages)
+        *   init session context (it will be shared for all following steps)
+        *   iterate over all light frame sessions in the current selection
+        *     for each session:
+        *       update context input and output files
+        *       run session.light stages
+        *   after all sessions are processed, run final.stack stages (using the shared context and temp dir)
+
+        """
         logging.info("--- Running all stages ---")
 
         # 1. Get all pipeline definitions (the `[[stages]]` tables with name and priority).
@@ -700,6 +742,89 @@ class Starbash:
         for task in tasks_to_run:
             self.run_stage(task)
 
+    def get_recipe_for_session(
+        self, session: SessionRow, step: dict[str, Any]
+    ) -> Repo | None:
+        """Try to find a recipe that can be used to process the given session for the given step name
+        (master-dark, master-bias, light, stack, etc...)
+
+        * if a recipe doesn't have a matching recipe.stage.<step_name> it is not considered
+        * As part of this checking we will look at recipe.auto.require.* conditions to see if the recipe
+        is suitable for this session.
+        * the imagetyp of this session matches step.input
+
+        Currently we return just one Repo but eventually we should support multiple matching recipes
+        and make the user pick (by throwing an exception?).
+        """
+        # Get all recipe repos - FIXME add a getall(kind) to RepoManager
+        recipe_repos = [r for r in self.repo_manager.repos if r.kind() == "recipe"]
+
+        step_name = step.get("name")
+        if not step_name:
+            raise ValueError("Invalid pipeline step found: missing 'name' key.")
+
+        input_name = step.get("input")
+        if not input_name:
+            raise ValueError("Invalid pipeline step found: missing 'input' key.")
+
+        imagetyp = session.get(get_column_name(Database.IMAGETYP_KEY))
+
+        if not imagetyp or input_name != self.aliases.normalize(imagetyp):
+            logging.debug(
+                f"Session imagetyp '{imagetyp}' does not match step input '{input_name}', skipping"
+            )
+            return None
+
+        # Get session metadata for checking requirements
+        session_metadata = session.get("metadata", {})
+
+        for repo in recipe_repos:
+            # Check if this recipe has the requested stage
+            stage_config = repo.get(f"recipe.stage.{step_name}")
+            if not stage_config:
+                logging.debug(
+                    f"Recipe {repo.url} does not have stage '{step_name}', skipping"
+                )
+                continue
+
+            # Check auto.require conditions if they exist
+
+            # If requirements are specified, check if session matches
+            required_filters = repo.get("auto.require.filter", [])
+            if required_filters:
+                session_filter = self.aliases.normalize(
+                    session_metadata.get(Database.FILTER_KEY)
+                )
+
+                # Session must have a filter that matches one of the required filters
+                if not session_filter or session_filter not in required_filters:
+                    logging.debug(
+                        f"Recipe {repo.url} requires filters {required_filters}, "
+                        f"session has '{session_filter}', skipping"
+                    )
+                    continue
+
+            required_cameras = repo.get("auto.require.camera", [])
+            if required_cameras:
+                session_camera = self.aliases.normalize(
+                    session_metadata.get("INSTRUME")
+                )  # Camera identifier
+
+                # Session must have a camera that matches one of the required cameras
+                if not session_camera or session_camera not in required_cameras:
+                    logging.debug(
+                        f"Recipe {repo.url} requires cameras {required_cameras}, "
+                        f"session has '{session_camera}', skipping"
+                    )
+                    continue
+
+            # This recipe matches!
+            logging.info(f"Selected recipe {repo.url} for stage '{step_name}' ")
+            return repo
+
+        # No matching recipe found
+        return None
+
     def run_master_stages(self):
         """Generate any missing master frames
 
@@ -711,62 +836,29 @@ class Starbash:
         *    add_input_to_context() add the input files to the context (from the session)
         *    run_stage(task) to generate the new master frame
         """
+        sorted_pipeline = self._get_stages("master-stages")
         sessions = self.search_session()
-        for session in sessions:
-            try:
-                imagetyp = session[get_column_name(Database.IMAGETYP_KEY)]
-                logging.debug(
-                    f"Processing session ID {session[get_column_name(Database.ID_KEY)]} with imagetyp '{imagetyp}'"
-                )
-
-                sorted_pipeline = self._get_stages("master-stages")
-
-                # 4. Iterate through the sorted pipeline and execute the associated tasks.
-                # FIXME unify the master vs normal step running code
-                for step in sorted_pipeline:
-                    step_name = step.get("name")
-                    if not step_name:
-                        raise ValueError(
-                            "Invalid pipeline step found: missing 'name' key."
-                        )
-
-                    # 3. Get all available task definitions (the `[[stage]]` tables with tool, script, when).
-                    task_definitions = self.repo_manager.merged.getall("stage")
-                    all_tasks = list(itertools.chain.from_iterable(task_definitions))
-
-                    # Find all tasks that should run during this step
-                    tasks_to_run = [
-                        task for task in all_tasks if task.get("when") == step_name
-                    ]
-
-                    for task in tasks_to_run:
-                        input_config = task.get("input", {})
-                        input_type = input_config.get("type")
-                        if not input_type:
-                            raise ValueError(
-                                f"Task for step '{step_name}' missing required input.type"
-                            )
-                        if self.aliases.equals(input_type, imagetyp):
-                            logging.debug(
-                                f"Running {step_name} task for imagetyp '{imagetyp}'"
-                            )
-
-                            # Create a default process dir in /tmp, though more advanced 'session' based workflows will
-                            # probably override this and place it somewhere persistent.
-                            with tempfile.TemporaryDirectory(
-                                prefix="session_tmp_"
-                            ) as temp_dir:
-                                logging.debug(
-                                    f"Created temporary session directory: {temp_dir}"
-                                )
-                                self.init_context()
-                                self.context["process_dir"] = temp_dir
-                                self.add_session_to_context(session)
-                                self.run_stage(task)
-            except RuntimeError as e:
-                logging.error(
-                    f"Skipping session {session[get_column_name(Database.ID_KEY)]}: {e}"
-                )
+        for session in track(sessions, description="Generating masters..."):
+            # 4. Iterate through the sorted pipeline and execute the associated tasks.
+            # FIXME unify the master vs normal step running code
+            for step in sorted_pipeline:
+                task = None
+                recipe = self.get_recipe_for_session(session, step)
+                if recipe:
+                    task = recipe.get("recipe.stage." + step["name"])
+
+                if task:
+                    input_config = task.get("input", {})
+                    input_type = input_config.get("type")
+                    if not input_type:
+                        raise ValueError(f"Task for step missing required input.type")
+
+                    # Create a default process dir in /tmp.
+                    # FIXME - eventually we should allow hashing or somesuch to keep reusing processing
+                    # dirs for particular targets?
+                    with ProcessingContext(self) as temp_dir:
+                        self.set_session_in_context(session)
+                        self.run_stage(task)
 
     def init_context(self) -> None:
         """Do common session init"""
@@ -780,8 +872,10 @@ class Starbash:
         }
         self.context.update(runtime_context)
 
-    def add_session_to_context(self, session: SessionRow) -> None:
+    def set_session_in_context(self, session: SessionRow) -> None:
         """adds to context from the indicated session:
+
+        Sets the following context variables based on the provided session:
         * instrument - for the session
         * date - the localtimezone date of the session
         * imagetyp - the imagetyp of the session
@@ -1000,7 +1094,7 @@ class Starbash:
             raise ValueError(
                 f"Tool '{tool_name}' for stage '{stage.get('name')}' not found."
             )
-        logging.debug(f"  Using tool: {tool_name}")
+        logging.debug(f"Using tool: {tool_name}")
         tool.set_defaults()
 
         # Allow stage to override tool timeout if specified

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/defaults/starbash.toml

@@ -23,6 +23,8 @@ HaOiii = ["HaOiii", "HaO3"]
 
 None = ["None"]
 
+camera_osc = ["OSC", "ZWO ASI2600MC Duo"]
+
 # Passes SII 672.4nm and H-Beta 486.1nm lines
 # Capturing of the two main emission wavebands in the deep red and blue at the same time
 #

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/recipes/master_bias/starbash.toml

@@ -7,7 +7,7 @@ kind = "recipe"
 author.name = "FIXMESiril?"
 author.email = "FIXMESiril?"
 
-[[stage]]
+[recipe.stage.master-bias]
 
 description = "Generate master bias"
 # disabled = false # turn on to skip
@@ -26,14 +26,12 @@ input.type = "bias" # look in all raw repos, but look only for bias files
 
 # Look for files in input repos, finding them by using the "relative" tag they contain
 input.source = "repo"
-input.required = 2 # siril needs at least 2 frames to stack
+input.required = 2    # siril needs at least 2 frames to stack
 # old school paths also work (but are not recommended)
 # input.path = ".../from_astroboy/masters-raw/2025-09-09/BIAS/*.fit*"
 
-when = "setup.master.bias" # run when master biases are regenerated
-
 # Based on the following definitions in the stage toml file...
-output.dest = "repo" # write to a particular repo
+output.dest = "repo"   # write to a particular repo
 output.type = "master" # write output to the special masters repo
 
 # the following fields will be auto populated in the context before entry:
@@ -65,4 +63,4 @@ script = '''
 
     # Stack frames
     stack frames rej 3 3 -nonorm -out={output["base_path"]}
-    '''
+    '''

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/recipes/master_dark/starbash.toml

@@ -7,7 +7,7 @@ kind = "recipe"
 author.name = "FIXMESiril?"
 author.email = "FIXMESiril?"
 
-[[stage]]
+[recipe.stage.master-dark]
 
 description = "Generate master dark"
 
@@ -20,8 +20,6 @@ input.type = "dark"
 input.source = "repo"
 input.required = 2    # siril needs at least 2 frames to stack
 
-when = "setup.master.dark" # run when master darks are regenerated
-
 # Based on the following definitions in the stage toml file...
 output.dest = "repo"   # write to a particular repo
 output.type = "master" # write output to the special masters repo

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/recipes/master_flat/starbash.toml

@@ -8,7 +8,7 @@ author.name = "FIXMESiril?"
 author.email = "FIXMESiril?"
 
 
-[[stage]]
+[recipe.stage.master-flat]
 
 # See master_bias/starbash.toml for more documentation
 
@@ -25,16 +25,14 @@ input.type = "flat" # look in all raw repos, but look only for flat files
 
 # Look for files in input repos, finding them by using the "relative" tag they contain
 input.source = "repo"
-input.required = 2 # siril needs at least 2 frames to stack
+input.required = 2    # siril needs at least 2 frames to stack
 
 # We require a master bias frame for this recipe.  By the time our recipe is invoked
 # context.master.bias will have been set to a full path to a master bias frame
 input.masters = ["bias"]
 
-when = "setup.master.flat" # run when master biases are regenerated
-
 # Based on the following definitions in the stage toml file...
-output.dest = "repo" # write to a particular repo
+output.dest = "repo"   # write to a particular repo
 output.type = "master" # write output to the special masters repo
 
 # FIXME for early development we have support for simple absolute file paths with globs

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/recipes/osc_dual_duo/starbash.toml

@@ -2,21 +2,10 @@
 [repo]
 kind = "recipe"
 
-# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
-# to the user and the file will be ignored for future processing.
-[recipe.require.version]
-min="0.1.0"
-max="4.5.8"
-
 [recipe]
 author.name = "Kevin Hester"
 author.email = "kevinh@geeksville.com"
 
-[[stage]]
-
-description = "Extract OSC dual duo filter Ha, Oiii and Sii channels"
-disabled = true # FIXME, debugging later stuff
-
 # FIXME-somehow-specify-what-filternames are used to auto detect this recipe can be used?
 # figure out how to support dual duo vs single duo.  Perhaps: the FIRST recipe that matches an auto rule
 # is used for any auto-defected defaults.  If an auto match is found it will be saved in the generated starter
@@ -25,27 +14,39 @@ disabled = true # FIXME, debugging later stuff
 # non OSC people use names like LRGB or SHO
 
 # for dual duo if we see Sii assume they also have HaOiii
-auto.for-filter = ["SiiOiii"]
+auto.require.filter = ["SiiOiii"]
 # for single duo look for this
-# auto.for-filter = ["HaOiii"]
-auto.for-camera = ["OSC"]
+# auto.require.filter = ["HaOiii"]
+auto.require.camera = ["OSC"]
+
+# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
+# to the user and the file will be ignored for future processing.
+[recipe.require.version]
+min="0.1.0"
+max="4.5.8"
+
+[recipe.stage.light]
+
+description = "Extract OSC dual duo filter Ha, Oiii and Sii channels"
+# disabled = true # FIXME, debugging later stuff
 
 tool.name = "siril"
 
 when = "session.light" # run once per session.config
-output = "FIXME"
+
+input.masters = ["bias", "flat"]
 
 # FIXME, bias and flat should have been added to context by two previous stages.  But for now hardwire
 # Note: they should not have filename extensions (see strip_extension in the old process.py)
-context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
+# context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
 
-context.sessionid = "2025-09-16" # FIXME, generate this by looping over all sessions (from outside this file)
-context.sessionconfig = "SiiOiii" # FIXME generate this by looping over all session configs
-context.light_base = "light_s{sessionid}_c{sessionconfig}"
+# context.sessionid = "2025-09-16" # FIXME, generate this by looping over all sessions (from outside this file)
+# context.sessionconfig = "SiiOiii" # FIXME generate this by looping over all session configs
+# context.light_base = "light_s{sessionid}_c{sessionconfig}"
 
 # FIXME until auto light finding is in
-input.source = "path"
-input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/LIGHT/*.fit*"
+# input.source = "path"
+# input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/LIGHT/*.fit*"
 
 script = '''
     # Create a sequence from the raw light frames, seq file goes to process_dir
@@ -64,7 +65,12 @@ script = '''
 
 temporaries = ["FIXME"]
 
-[[stage]]
+[recipe.stage.stack]
+
+disabled = false # not yet ready to test
+
+# FIXME this stage should only be considered if the previous stage in this same array
+# was run.  It must be run inside the same tempdir (so that files from previous stage are available)
 
 # FIXME, eventually we could make it optional to even have a starbash.toml.  If we find an
 # starbash.py we could introspect it for a starbash_config dict.  And look inside that for description
@@ -72,14 +78,17 @@ temporaries = ["FIXME"]
 
 description = "Stack OSC dual duo filter data, with separate Ha, Oiii and Sii channels"
 
-context.target = "M 27" # FIXME
-context.targets = "/workspaces/starbash/images/processed" # FIXME, do something smarter
+# context.target = "M 27" # FIXME
+# context.targets = "/workspaces/starbash/images/processed" # FIXME, do something smarter
 
 tool.name = "python"
 
-when = "session.stack" # run once after all session/session.config processing was done
+when = "final.stack" # run once after all session/session.config processing was done
 
-input.masters = ["bias", "flat"]
+# Based on the following definitions in the stage toml file...
+# FIXME, we should inherit this - most recipes shouldn't have to declare it
+output.dest = "repo" # write to a particular repo
+output.type = "processed" # write output to the special masters repo
 
 # if not specified starbash.py used
 # script-file = "script.py"

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/recipes/starbash.toml

@@ -32,28 +32,30 @@ dir = "osc_single_duo"
 # master specific stages
 #
 [[master-stages]]
-name = "setup.master.bias" # generate master bias frames
+name = "master-bias" # generate master bias frames
 priority = 10
+input = "bias" # only used for frames of this type
 
 [[master-stages]]
-name = "setup.master.dark" # generate master bias frames
+name = "master-dark" # generate master dark frames
 priority = 10
+input = "dark"
 
 [[master-stages]]
-name = "setup.master.flat" # generate master flat frames
+name = "master-flat" # generate master flat frames
 priority = 20
+input = "flat"
 
 #
-# session specific processing stages
+# session specific processing stages, not currently used, for now I just do this list from code
 #
-[[stages]]
-name = "session.config" # for flat processing, master generation etc
-priority = 10
 
 [[stages]]
-name = "session.light" # generate light frames from lights and with reference to flats/bias
+name = "light" # generate light frames from lights and with reference to flats/bias
 priority = 20
+input = "light" # only used for frames of this type
 
 [[stages]]
-name = "session.stack" # stack frames
+name = "stack" # stack frames
 priority = 30
+input = "light" # only used for frames of this type

{starbash-0.1.10 → starbash-0.1.11}/src/starbash/tool.py

@@ -3,11 +3,10 @@ import shutil
 import textwrap
 import tempfile
 import subprocess
-import select
 import re
-
+import threading
+import queue
 import logging
-
 import RestrictedPython
 
 logger = logging.getLogger(__name__)
@@ -194,33 +193,71 @@ def tool_run(
             pass
 
     # Stream output line by line in real-time
+    # Use threading for cross-platform compatibility (select doesn't work on Windows with pipes)
+
     assert process.stdout
     assert process.stderr
 
+    output_queue: queue.Queue = queue.Queue()
+
+    def read_stream(stream, log_func, stream_name):
+        """Read from stream and put lines in queue."""
+        try:
+            for line in stream:
+                line = line.rstrip("\n")
+                output_queue.put((log_func, stream_name, line))
+        finally:
+            output_queue.put((None, stream_name, None))  # Signal EOF
+
+    # Start threads to read stdout and stderr
+    stdout_thread = threading.Thread(
+        target=read_stream,
+        args=(process.stdout, logger.debug, "tool-stdout"),
+        daemon=True,
+    )
+    stderr_thread = threading.Thread(
+        target=read_stream,
+        args=(process.stderr, logger.warning, "tool-stderr"),
+        daemon=True,
+    )
+
+    stdout_thread.start()
+    stderr_thread.start()
+
+    # Track which streams have finished
+    streams_finished = 0
+
     try:
-        streams = {
-            process.stdout.fileno(): (process.stdout, logger.debug, "tool-stdout"),
-            process.stderr.fileno(): (process.stderr, logger.warning, "tool-stderr"),
-        }
-
-        while streams:
-            # Wait for data on any stream (with timeout for periodic checking)
-            ready, _, _ = select.select(list(streams.keys()), [], [], 0.1)
-
-            for fd in ready:
-                stream, log_func, stream_name = streams[fd]
-                line = stream.readline()
-
-                if line:
-                    # Strip trailing newline and log immediately
-                    line = line.rstrip("\n")
-                    log_func(f"[{stream_name}] {line}")
+        # Process output from queue until both streams are done
+        while streams_finished < 2:
+            try:
+                # Use timeout to periodically check if process has terminated
+                log_func, stream_name, line = output_queue.get(timeout=0.1)
+
+                if log_func is None:
+                    # EOF signal
+                    streams_finished += 1
                 else:
-                    # EOF reached on this stream
-                    del streams[fd]
+                    # Log the line
+                    log_func(f"[{stream_name}] {line}")
+
+            except queue.Empty:
+                # No output available, check if process terminated
+                if process.poll() is not None:
+                    # Process finished, wait a bit more for remaining output
+                    break
+
+        # Wait for threads to finish (they should be done or very close)
+        stdout_thread.join(timeout=1.0)
+        stderr_thread.join(timeout=1.0)
 
-            # Check if process has terminated
-            if process.poll() is not None and not streams:
+        # Drain any remaining items in queue
+        while not output_queue.empty():
+            try:
+                log_func, stream_name, line = output_queue.get_nowait()
+                if log_func is not None:
+                    log_func(f"[{stream_name}] {line}")
+            except queue.Empty:
                 break
 
         # Wait for process to complete with timeout