starbash 0.1.9__py3-none-any.whl → 0.1.15__py3-none-any.whl

starbash/recipes/README.md

@@ -1,3 +1,6 @@
 This is what a typical directory of recipes would look like.  it could be hosted locally in a directory tree, on github, whatever.
 
 Currently it lives in the starbash python blob, but eventually the 'master' set of recipes will live in a different repo.  In fact, different orgs could provide their own recipe repos.
+
+FIXME:
+For the time being we also have a few python scripts - until they are refactored a bit so they can live in the http tree.

starbash/recipes/master_bias/starbash.toml

@@ -7,16 +7,17 @@ kind = "recipe"
 author.name = "FIXMESiril?"
 author.email = "FIXMESiril?"
 
-[[stage]]
+[recipe.stage.master-bias]
 
 description = "Generate master bias"
-disabled = false # FIXME, debugging later stuff
+# disabled = false # turn on to skip
 
 # Restrict processing of this stage to only if detected hardware was found for this session
 # For any camera
-auto.for-camera = []
+# auto.for-camera = []
 
-tool = "siril"
+tool.name = "siril"
+# tool.timeout = 15.0 # allow up to 15 seconds before we timeout and kill tool
 
 # or auto?
 # find the most recent raw fits for the current instrument (as of the time of session start)
@@ -25,25 +26,21 @@ input.type = "bias" # look in all raw repos, but look only for bias files
 
 # Look for files in input repos, finding them by using the "relative" tag they contain
 input.source = "repo"
+input.required = 2    # siril needs at least 2 frames to stack
+# old school paths also work (but are not recommended)
 # input.path = ".../from_astroboy/masters-raw/2025-09-09/BIAS/*.fit*"
-input.required = true # Is at least one input file required?  If true, we will skip running this stage entirely (with a warning)
-
-# make the following also work
-#
-#os.makedirs(os.path.dirname(output), exist_ok=True)
-#os.makedirs(os.path.dirname(process_dir), exist_ok=True)
-#frames = glob(f"{masters_raw}/{date}/BIAS/{date}_*.fit*")
-#siril_run_in_temp_dir(frames, ...
-when = "setup.masters" # run when master biases are regenerated
 
 # Based on the following definitions in the stage toml file...
-output.dest = "repo" # write to a particular repo
+output.dest = "repo"   # write to a particular repo
 output.type = "master" # write output to the special masters repo
 
+
+# context.target - the name of the target (m31 etc...) we are processing (if available)
 # the following fields will be auto populated in the context before entry:
 # context.output.base_path - the full filepath to write the output file to **excluding the suffix**
 # context.output.full_path - the full filepath to write the output file to (including suffix)
-# (NOT implemented / needed) context.output.root_path - points to the base of the destination repo
+# context.output.repo - points to the destination repo object
+# context.output.relative_base_path - the relative path within the repo (excluding suffix) (for logging/user message purposes)
 # (NOT implemented / needed) context.output.suffix - the suffix to append to the output file (e.g. .fits or .fit.gz)
 
 # The following constants are auto defined before running the tool
@@ -64,9 +61,9 @@ output.type = "master" # write output to the special masters repo
 
 script = '''
     # Convert Bias Frames to .fit files
-    link bias -out={process_dir}
+    link frames -out={process_dir}
     cd {process_dir}
 
-    # Stack Bias Frames to bias_stacked.fit
-    stack bias rej 3 3 -nonorm -out={output["base_path"]}
-    '''
+    # Stack frames
+    stack frames rej 3 3 -nonorm -out={output["base_path"]}
+    '''

starbash/recipes/master_dark/starbash.toml

@@ -0,0 +1,33 @@
+
+[repo]
+kind = "recipe"
+
+
+[recipe]
+author.name = "FIXMESiril?"
+author.email = "FIXMESiril?"
+
+[recipe.stage.master-dark]
+
+description = "Generate master dark"
+
+tool.name = "siril"
+
+input.type = "dark"
+
+# Look for files in input repos, finding them by using the "relative" tag they contain
+input.source = "repo"
+input.required = 2    # siril needs at least 2 frames to stack
+
+# Based on the following definitions in the stage toml file...
+output.dest = "repo"   # write to a particular repo
+output.type = "master" # write output to the special masters repo
+
+script = '''
+    # Convert bias/dark Frames to .fit files
+    link frames -out={process_dir}
+    cd {process_dir}
+
+    # Stack frames
+    stack frames rej 3 3 -nonorm -out={output["base_path"]}
+    '''

starbash/recipes/master_flat/starbash.toml

@@ -8,28 +8,36 @@ author.name = "FIXMESiril?"
 author.email = "FIXMESiril?"
 
 
-[[stage]]
+[recipe.stage.master-flat]
+
+# See master_bias/starbash.toml for more documentation
 
 description = "Generate master flat"
-disabled = true # FIXME, debugging later stuff
+# disabled = false
 
-# For any camera
-auto.for-camera = []
+tool.name = "siril"
+# tool.timeout = 15.0 # allow up to 15 seconds before we timeout and kill tool
 
-tool = "siril"
-# input.source = "session" # or auto? prefer ones in session otherwise find by in masters
-input.type = "flat" # look in _session_ directories, but look only for flat files
+# or auto?
+# find the most recent raw fits for the current instrument (as of the time of session start)
+# input.source = "most-recent" # only look for the most recent set of raws for this particular type
+input.type = "flat" # look in all raw repos, but look only for flat files
 
-# FIXME for early development we have support for simple absolute file paths with globs
-input.source = "path"
-input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/FLAT/*.fit*"
-input.required = true # Is at least one input file required?  If true, we will skip running this stage entirely (with a warning)
+# Look for files in input repos, finding them by using the "relative" tag they contain
+input.source = "repo"
+input.required = 2    # siril needs at least 2 frames to stack
 
-when = "session-config" # run once per session-config
-context.output = "{process_dir}/flat_s{sessionid}_c{sessionconfig}.fits"
+# We require a master bias frame for this recipe.  By the time our recipe is invoked
+# context.master.bias will have been set to a full path to a master bias frame
+input.masters = ["bias"]
 
-# FIXME, bias should have been added to context by two previous stages.  But for now hardwire
-context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
+# Based on the following definitions in the stage toml file...
+output.dest = "repo"   # write to a particular repo
+output.type = "master" # write output to the special masters repo
+
+# FIXME for early development we have support for simple absolute file paths with globs
+#input.source = "path"
+#input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/FLAT/*.fit*"
 
 script = '''
     # Create a sequence from the raw flat frames
@@ -37,10 +45,10 @@ script = '''
     cd {process_dir}
 
     # Calibrate the flat frames using master bias
-    calibrate flat -bias={bias}
+    calibrate flat -bias={master["bias"]}
 
-    # Stack the pre-processed (calibrated) flat frames (writes to flat_stacked.fit)
-    stack pp_flat rej 3 3 -norm=mul -out=flat_stacked
+    # Stack the pre-processed (calibrated) flat frames
+    stack pp_flat rej 3 3 -norm=mul -out={output["base_path"]}
     '''
 
 temporaries = ["flat", "pp_flat"]

starbash/recipes/osc.py

@@ -0,0 +1,190 @@
+# pyright: reportUndefinedVariable=false
+
+
+import logging
+import os
+from glob import glob
+from typing import Any
+
+from starbash.processing import NotEnoughFilesError
+from starbash.tool import tools
+
+siril = tools["siril"]
+
+delete_temps = False
+
+# ('context' and 'logger' are normally injected by the starbash runtime)
+context: dict[str, Any] = {}
+logger: logging.Logger = None  # type: ignore
+
+
+# FIXME move this into main starbash
+def perhaps_delete_temps(temps: list[str]) -> None:
+    if delete_temps:
+        for t in temps:
+            for path in glob(f"{context['process_dir']}/{t}_*"):
+                os.remove(path)
+
+
+def make_stacked(variant: str | None, output_file: str):
+    """
+    Registers and stacks all pre-processed light frames for a given filter configuration
+    across all sessions.
+    """
+    # If we are being invoked for simple/non duo filter we won't have variants generated by prior steps in the workflow
+    if variant is None:
+        input_base = "bkg_pp_light"
+    else:
+        input_base = f"{variant}_bkg_pp_light"
+
+    # The sequence name for all frames of this variant across all sessions
+    # e.g. Ha_bkg_pp_light_cHaOiii
+    merged_seq_base = f"all_{input_base}"
+
+    # Absolute path for the output stacked file
+    stacked_output_path = glob(f"{context['process_dir']}/{output_file}.fit*")
+
+    if stacked_output_path:
+        logger.info(f"Using existing stacked file: {stacked_output_path}")
+    else:
+        # Merge all frames (from multiple sessions and configs) use those for stacking
+        frames = glob(f"{context['process_dir']}/{input_base}_s*.fit*")
+
+        logger.info(
+            f"Registering and stacking {len(frames)} frames for {variant} -> {stacked_output_path}"
+        )
+        if len(frames) < 2:
+            raise NotEnoughFilesError("Need at least two frames", frames)
+
+        # Siril commands for registration and stacking. We run this in process_dir.
+        commands = f"""
+            link {merged_seq_base} -out={context["process_dir"]}
+            cd {context["process_dir"]}
+
+            # We use -2pass to select the best possible reference frame for others to register against
+            register {merged_seq_base} -2pass
+
+            # because we are using -2pass we must complete the registration here before stacking
+            # FIXME make drizzle optional
+            seqapplyreg {merged_seq_base} -drizzle
+
+            stack r_{merged_seq_base} rej g 0.3 0.05 -filter-wfwhm=3k -norm=addscale -output_norm -32b -out={output_file}
+
+            # and flip if required
+            mirrorx_single {output_file}
+            """
+
+        context["input_files"] = frames
+        siril.run(commands, context=context)
+
+    perhaps_delete_temps([merged_seq_base, f"r_{merged_seq_base}"])
+
+
+def make_renormalize(channel_num: int):
+    """
+    Aligns the stacked images (Sii, Ha, OIII) and renormalizes Sii and OIII
+    to match the flux of the Ha channel.
+    """
+    logger.info("Aligning and renormalizing stacked images.")
+
+    # Define file basenames for the stacked images created in the 'process' directory
+    ha_base = "results_00001"
+    oiii_base = "results_00002"
+    sii_base = "results_00003"
+
+    # Define final output paths. The 'results' directory is a symlink in the work dir.
+    results_dir = context["output"]["base_path"]
+    os.makedirs(results_dir, exist_ok=True)
+
+    commands = ""
+
+    if channel_num == 1:
+        # Only one channel - just copy it - eventually we'll add other metadata
+        final_path = f"{results_dir}/stacked.fits"
+        commands += f"""
+            load results_00001
+            save "{final_path}"
+            """
+
+    # Basenames for registered files (output of 'register' command)
+    r_ha = f"r_{ha_base}"
+    r_oiii = f"r_{oiii_base}"
+
+    if channel_num >= 2:
+        # Do pixelmath to fixup channel brightness
+        logger.info("Doing renormalisation of extra Ha/Oiii channels")
+
+        ha_final_path = f"{results_dir}/stacked_Ha.fits"
+        oiii_final_path = f"{results_dir}/stacked_OIII.fits"
+
+        # Pixel math formula for renormalization.
+        # It matches the median and spread (MAD) of a channel to a reference channel (Ha).
+        # Formula: new = old * (MAD(ref)/MAD(old)) - (MAD(ref)/MAD(old)) * MEDIAN(old) + MEDIAN(ref)
+        pm_oiii = f'"${r_oiii}$*mad(${r_ha}$)/mad(${r_oiii}$)-mad(${r_ha}$)/mad(${r_oiii}$)*median(${r_oiii}$)+median(${r_ha}$)"'
+
+        # Siril commands to be executed in the 'process' directory
+        commands += f"""
+            # -transf=shift fails sometimes, which I guess is possible because we have multiple sessions with possible different camera rotation
+            # -interp=none also fails sometimes, so let default interp happen
+            # -drizzle is required for success on many images
+            register results -drizzle
+            pm {pm_oiii}
+            update_key FILTER Oiii "OSC Duo filter extracted"
+            save "{oiii_final_path}"
+            load {r_ha}
+            update_key FILTER Ha "OSC Duo filter extracted"
+            save "{ha_final_path}"
+            """
+
+    if channel_num >= 3:
+        logger.info("Doing renormalisation of extra Sii channel")
+
+        sii_final_path = f"{results_dir}/stacked_Sii.fits"
+        r_sii = f"r_{sii_base}"
+        pm_sii = f'"${r_sii}$*mad(${r_ha}$)/mad(${r_sii}$)-mad(${r_ha}$)/mad(${r_sii}$)*median(${r_sii}$)+median(${r_ha}$)"'
+        commands += f"""
+            pm {pm_sii}
+            update_key FILTER Sii "OSC dual Duo filter extracted"
+            save "{sii_final_path}"
+            """
+
+    siril.run(commands, context=context, cwd=context["process_dir"])
+    logger.info(f"Saved final renormalized images to {results_dir}")
+
+
+def osc_process(has_ha_oiii: bool, has_sii_oiii: bool):
+    """Shared code for use by OSC processing scripts"""
+
+    logger.info(f"Running osc_process(has_ha_oiii={has_ha_oiii}, has_sii_oiii={has_sii_oiii})")
+    logger.debug("Using context: %s", context)
+
+    channel_num = 0
+    if has_sii_oiii:
+        # red output channel - from the SiiOiii filter Sii is on the 672nm red channel (mistakenly called Ha by siril)
+        channel_num += 1
+        make_stacked("Ha", f"results_{channel_num:05d}")
+
+    if has_ha_oiii:
+        # green output channel - from the HaOiii filter Ha is on the 656nm red channel
+        channel_num += 1
+        make_stacked("Ha", f"results_{channel_num:05d}")
+
+    if has_ha_oiii or has_sii_oiii:
+        # blue output channel - both filters have Oiii on the 500nm blue channel.  Note the case here is uppercase to match siril output
+        channel_num += 1
+        make_stacked("OIII", f"results_{channel_num:05d}")
+
+    # if we haven't already processed some other way - just do a single channel process
+    # FIXME in this case we want to use a siril line like "stack r_bkg_pp_light rej g 0.3 0.05 -filter-wfwhm=3k -norm=addscale -output_norm -rgb_equal -32b -out=result"
+    if channel_num == 0:
+        # single channel - just stack all Ha frames together
+        channel_num += 1
+        make_stacked(None, f"results_{channel_num:05d}")
+
+    # There might be an old/state autogenerated .seq file, delete it so it doesn't confuse renormalize
+    results_seq_path = f"{context['process_dir']}/results_.seq"
+    if os.path.exists(results_seq_path):
+        os.remove(results_seq_path)
+
+    assert channel_num >= 1, "At least one channel should have been processed"
+    make_renormalize(channel_num)

starbash/recipes/osc_dual_duo/starbash.toml

@@ -2,20 +2,13 @@
 [repo]
 kind = "recipe"
 
-# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
-# to the user and the file will be ignored for future processing.
-[recipe.require.version]
-min="0.1.0"
-max="4.5.8"
-
 [recipe]
-author.name = "Kevin Hester"
-author.email = "kevinh@geeksville.com"
+author.name = "FIXMEsirilsomeone"
+author.email = "FIXMEsirilsomeone"
 
-[[stage]]
-
-description = "Extract OSC dual duo filter Ha, Oiii and Sii channels"
-disabled = true # FIXME, debugging later stuff
+# Lower priority recipes are tried to match first
+# - we want this to match after the osc_dual_duo has a chance
+priority = 10
 
 # FIXME-somehow-specify-what-filternames are used to auto detect this recipe can be used?
 # figure out how to support dual duo vs single duo.  Perhaps: the FIRST recipe that matches an auto rule
@@ -25,26 +18,39 @@ disabled = true # FIXME, debugging later stuff
 # non OSC people use names like LRGB or SHO
 
 # for dual duo if we see Sii assume they also have HaOiii
-auto.for-filter = ["SiiOiii"]
+auto.require.filter = ["SiiOiii"]
 # for single duo look for this
-# auto.for-filter = ["HaOiii"]
-auto.for-camera = ["OSC"]
+# auto.require.filter = ["HaOiii"]
+
+# To require a color camera use this.  or to require a mono camera auto.require.mono.
+auto.require.color = true
+
+# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
+# to the user and the file will be ignored for future processing.
+[recipe.require.version]
+min="0.1.0"
+max="4.5.8"
+
+[recipe.stage.light]
 
-tool = "siril"
-when = "session-light" # run once per session-config
-output = "FIXME"
+description = "Extract OSC dual duo filter Ha, Oiii and Sii channels"
+# disabled = true # FIXME, debugging later stuff
 
-# FIXME, bias and flat should have been added to context by two previous stages.  But for now hardwire
-# Note: they should not have filename extensions (see strip_extension in the old process.py)
-context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
+tool.name = "siril"
+
+# Auto find input light frames for the current session
+# Look for files in input repos, finding them by using the "relative" tag they contain
+input.source = "repo"
+input.required = 2    # siril needs at least 2 frames to stack
+input.type = "light" # look in all raw repos, but look only for light files
+
+# Auto find suitable masters of the following type
+input.masters = ["bias", "flat"]
+
+# the base name for our light files
+context.light_base = '''light_s{session["id"]}'''
 
-context.sessionid = "2025-09-16" # FIXME, generate this by looping over all sessions (from outside this file)
-context.sessionconfig = "SiiOiii" # FIXME generate this by looping over all session configs
-context.light_base = "light_s{sessionid}_c{sessionconfig}"
 
-# FIXME until auto light finding is in
-input.source = "path"
-input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/LIGHT/*.fit*"
 
 script = '''
     # Create a sequence from the raw light frames, seq file goes to process_dir
@@ -52,7 +58,7 @@ script = '''
     cd {process_dir}
 
     # Calibrate the light frames using master bias and flat
-    calibrate {light_base} -bias={bias} -flat=flat -cfa -equalize_cfa
+    calibrate {light_base} -bias={master["bias"]} -flat={master["flat"]} -cfa -equalize_cfa
 
     # Remove background gradient on a per-frame basis (generates bkg_pp_light.seq)
     seqsubsky pp_{light_base} 1
@@ -61,28 +67,32 @@ script = '''
     seqextract_HaOIII bkg_pp_{light_base} -resample=ha
     '''
 
-temporaries = ["FIXME"]
+# temporaries = ["FIXME"]
 
-[[stage]]
+[recipe.stage.stack]
 
-# FIXME, eventually we could make it optional to even have a starbash.toml.  If we find an
-# starbash.py we could introspect it for a starbash_config dict.  And look inside that for description
-# "stage", "when" or whatever?
+# this stage is considered if the previous stage in this same array
+# was run.  It must be run inside the same tempdir (so that files from previous stage are available)
 
-description = "Stack OSC dual duo filter data, with separate Ha, Oiii and Sii channels"
+description = "Stack OSC dual duo (HaOiii + SiiOiii) filter data: with separate Ha, Oiii and Sii channels"
 
-context.target = "M 27" # FIXME
-context.targets = "/workspaces/starbash/images/processed" # FIXME, do something smarter
+input.source = "recipe" # we will use output files from previous stages in this same recipe as our input
 
-tool = "python"
-when = "session-stack" # run once after all session/session-config processing was done
+tool.name = "python"
+
+# Based on the following definitions in the stage toml file...
+# FIXME, we should inherit this - most recipes shouldn't have to declare it
+output.dest = "repo" # write to a particular repo
+output.type = "processed" # write output to the special processed repo
 
 # if not specified starbash.py used
 # script-file = "script.py"
 
-# or inline python code instead of that function?
-#script = '''
-#    make_stacked("SiiOiii", "Ha", f"results_00001")
-#    ...
-#    (moved to script.py)
-#    '''
+# or inline python code can be provided here.  In this case I'm using some python
+# code I'm temporarily sharing from the main project...
+script = '''
+    from starbash.recipes import osc
+    osc.logger = logger
+    osc.context = context
+    osc.osc_process(has_ha_oiii=True, has_sii_oiii=True)
+    '''

starbash/recipes/osc_simple/starbash.toml

@@ -0,0 +1,82 @@
+
+[repo]
+kind = "recipe"
+
+[recipe]
+
+# Lower priority recipes are tried to match first
+# - we want this to match after the osc_dual_duo has a chance
+priority = 30
+
+author.name = "FIXMEsirilsomeone"
+author.email = "FIXMEsirilsomeone"
+
+# no filter requirements - will work for any osc session
+auto.require.color = true
+
+# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
+# to the user and the file will be ignored for future processing.
+[recipe.require.version]
+min="0.1.0"
+max="4.5.8"
+
+[recipe.stage.light]
+
+description = "Calibrate OSC frames vs bias and flats"
+# disabled = true # FIXME, debugging later stuff
+
+tool.name = "siril"
+
+# Auto find input light frames for the current session
+# Look for files in input repos, finding them by using the "relative" tag they contain
+input.source = "repo"
+input.required = 2    # siril needs at least 2 frames to stack
+input.type = "light" # look in all raw repos, but look only for light files
+
+# Auto find suitable masters of the following type
+input.masters = ["bias", "flat"]
+
+# the base name for our light files
+context.light_base = '''light_s{session["id"]}'''
+
+script = '''
+    # Create a sequence from the raw light frames, seq file goes to process_dir
+    link {light_base} -out={process_dir}
+    cd {process_dir}
+
+    # Calibrate the light frames using master bias and flat
+    calibrate {light_base} -bias={master["bias"]} -flat={master["flat"]} -cfa -equalize_cfa
+
+    # Remove background gradient on a per-frame basis (generates bkg_pp_light.seq)
+    seqsubsky pp_{light_base} 1
+    '''
+
+# temporaries = ["FIXME"]
+
+[recipe.stage.stack]
+
+# this stage is only be considered if the previous stage in this same array
+# was run.  It must be run inside the same tempdir (so that files from previous stage are available)
+
+description = "Stack OSC simple (non duo filter) images"
+
+input.source = "recipe" # we will use output files from previous stages in this same recipe as our input
+
+tool.name = "python"
+
+# Based on the following definitions in the stage toml file...
+# FIXME, we should inherit this - most recipes shouldn't have to declare it
+output.dest = "repo" # write to a particular repo
+output.type = "processed" # write output to the special processed repo
+
+# if not specified starbash.py used
+# script-file = "script.py"
+
+# or inline python code can be provided here.  In this case I'm using some python
+# code I'm temporarily sharing from the main project...
+script = '''
+    from starbash.recipes import osc
+    osc.logger = logger
+    osc.context = context
+    osc.osc_process(has_ha_oiii=False, has_sii_oiii=False)
+    '''