starbash 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

starbash/recipes/master_flat/starbash.toml

@@ -0,0 +1,46 @@
+
+[repo]
+kind = "recipe"
+
+
+[recipe]
+author.name = "FIXMESiril?"
+author.email = "FIXMESiril?"
+
+
+[[stage]]
+
+description = "Generate master flat"
+disabled = true # FIXME, debugging later stuff
+
+# For any camera
+auto.for-camera = []
+
+tool = "siril"
+# input.source = "session" # or auto? prefer ones in session otherwise find by in masters
+input.type = "flat" # look in _session_ directories, but look only for flat files
+
+# FIXME for early development we have support for simple absolute file paths with globs
+input.source = "path"
+input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/FLAT/*.fit*"
+input.required = true # Is at least one input file required?  If true, we will skip running this stage entirely (with a warning)
+
+when = "session-config" # run once per session-config
+context.output = "{process_dir}/flat_s{sessionid}_c{sessionconfig}.fits"
+
+# FIXME, bias should have been added to context by two previous stages.  But for now hardwire
+context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
+
+script = '''
+    # Create a sequence from the raw flat frames
+    link flat -out={process_dir}
+    cd {process_dir}
+
+    # Calibrate the flat frames using master bias
+    calibrate flat -bias={bias}
+
+    # Stack the pre-processed (calibrated) flat frames (writes to flat_stacked.fit)
+    stack pp_flat rej 3 3 -norm=mul -out=flat_stacked
+    '''
+
+temporaries = ["flat", "pp_flat"]

starbash/recipes/osc_dual_duo/starbash.py

@@ -0,0 +1,151 @@
+# pyright: reportUndefinedVariable=false
+# ('context' and 'logger' are injected by the starbash runtime)
+
+import os
+from glob import glob
+from starbash.tool import tools
+
+siril = tools["siril"]
+
+delete_temps = False
+
+
+# FIXME move this into main starbash
+def perhaps_delete_temps(temps: list[str]) -> None:
+    if delete_temps:
+        for t in temps:
+            for path in glob(f"{context['process_dir']}/{t}_*"):
+                os.remove(path)
+
+
+def normalize_target_name(name: str) -> str:
+    """Converts a target name to an any filesystem-safe format by removing spaces"""
+    return name.replace(" ", "").upper()
+
+
+def make_stacked(sessionconfig: str, variant: str, output_file: str):
+    """
+    Registers and stacks all pre-processed light frames for a given filter configuration
+    across all sessions.
+    """
+    # The sequence name for all frames of this variant across all sessions
+    # e.g. Ha_bkg_pp_light_cHaOiii
+    merged_seq_base = f"all_{variant}_bkg_pp_light"
+
+    # Absolute path for the output stacked file
+    stacked_output_path = glob(f"{context["process_dir"]}/{output_file}.fit*")
+
+    if stacked_output_path:
+        logger.info(f"Using existing stacked file: {stacked_output_path}")
+    else:
+        # Merge all frames (from multiple sessions and configs) use those for stacking
+        frames = glob(
+            f"{context["process_dir"]}/{variant}_bkg_pp_light_s*_c{sessionconfig}_*.fit*"
+        )
+
+        logger.info(
+            f"Registering and stacking {len(frames)} frames for {sessionconfig}/{variant} -> {stacked_output_path}"
+        )
+
+        # Siril commands for registration and stacking. We run this in process_dir.
+        commands = f"""
+            link {merged_seq_base} -out={context["process_dir"]}
+            cd {context["process_dir"]}
+
+            register {merged_seq_base}
+            stack r_{merged_seq_base} rej g 0.3 0.05 -filter-wfwhm=3k -norm=addscale -output_norm -32b -out={output_file}
+
+            # and flip if required
+            mirrorx_single {output_file}
+            """
+
+        context["input_files"] = frames
+        siril.run_in_temp_dir(commands, context=context)
+
+    perhaps_delete_temps([merged_seq_base, f"r_{merged_seq_base}"])
+
+
+def make_renormalize():
+    """
+    Aligns the stacked images (Sii, Ha, OIII) and renormalizes Sii and OIII
+    to match the flux of the Ha channel.
+    """
+    logger.info("Aligning and renormalizing stacked images.")
+
+    # Define file basenames for the stacked images created in the 'process' directory
+    ha_base = "results_00001"
+    oiii_base = "results_00002"
+    sii_base = "results_00003"
+
+    # Define final output paths. The 'results' directory is a symlink in the work dir.
+    results_dir = f"{context["targets"]}/{normalize_target_name(context["target"])}"
+    os.makedirs(results_dir, exist_ok=True)
+
+    ha_final_path = f"{results_dir}/stacked_Ha.fits"
+    oiii_final_path = f"{results_dir}/stacked_OIII.fits"
+
+    # Check if final files already exist to allow resuming
+    if all(os.path.exists(f) for f in [ha_final_path, oiii_final_path]):
+        logger.info("Renormalized files already exist, skipping.")
+        return
+
+    # Basenames for registered files (output of 'register' command)
+    r_ha = f"r_{ha_base}"
+    r_oiii = f"r_{oiii_base}"
+
+    # Pixel math formula for renormalization.
+    # It matches the median and spread (MAD) of a channel to a reference channel (Ha).
+    # Formula: new = old * (MAD(ref)/MAD(old)) - (MAD(ref)/MAD(old)) * MEDIAN(old) + MEDIAN(ref)
+    pm_oiii = f'"${r_oiii}$*mad(${r_ha}$)/mad(${r_oiii}$)-mad(${r_ha}$)/mad(${r_oiii}$)*median(${r_oiii}$)+median(${r_ha}$)"'
+
+    # Siril commands to be executed in the 'process' directory
+    commands = f"""
+        # -transf=shift fails sometimes, which I guess is possible because we have multiple sessions with possible different camera rotation
+        # -interp=none also fails sometimes, so let default interp happen
+        register results
+        pm {pm_oiii}
+        update_key FILTER Oiii "OSC dual Duo filter extracted"
+        save "{oiii_final_path}"
+        load {r_ha}
+        update_key FILTER Ha "OSC dual Duo filter extracted"
+        save "{ha_final_path}"
+        """
+
+    if os.path.exists(f"{results_dir}/{sii_base}.fit"):
+        logger.info(f"Doing renormalisation of extra Sii channel")
+
+        sii_final_path = f"{results_dir}/stacked_Sii.fits"
+        r_sii = f"r_{sii_base}"
+        pm_sii = f'"${r_sii}$*mad(${r_ha}$)/mad(${r_sii}$)-mad(${r_ha}$)/mad(${r_sii}$)*median(${r_sii}$)+median(${r_ha}$)"'
+        commands += f"""
+            pm {pm_sii}
+            update_key FILTER Sii "OSC dual Duo filter extracted"
+            save "{sii_final_path}"
+            """
+
+    siril.run(context["process_dir"], commands, context=context)
+    logger.info(f"Saved final renormalized images to {results_dir}")
+
+
+def osc_dual_duo_post_session():
+    logger.info("Running osc_dual_duo_post_session python script")
+    logger.info("Using context: %s", context)
+
+    # red output channel - from the SiiOiii filter Sii is on the 672nm red channel (mistakenly called Ha by siril)
+    make_stacked("SiiOiii", "Ha", f"results_00001")
+
+    # green output channel - from the HaOiii filter Ha is on the 656nm red channel
+    make_stacked("HaOiii", "Ha", f"results_00001")
+
+    # blue output channel - both filters have Oiii on the 500nm blue channel.  Note the case here is uppercase to match siril output
+    make_stacked("*", "OIII", f"results_00002")
+
+    # There might be an old/state autogenerated .seq file, delete it so it doesn't confuse renormalize
+    results_seq_path = f"{context["process_dir"]}/results_.seq"
+    if os.path.exists(results_seq_path):
+        os.remove(results_seq_path)
+
+    make_renormalize()
+
+
+osc_dual_duo_post_session()

starbash/recipes/osc_dual_duo/starbash.toml

@@ -0,0 +1,88 @@
+
+[repo]
+kind = "recipe"
+
+# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
+# to the user and the file will be ignored for future processing.
+[recipe.require.version]
+min="0.1.0"
+max="4.5.8"
+
+[recipe]
+author.name = "Kevin Hester"
+author.email = "kevinh@geeksville.com"
+
+[[stage]]
+
+description = "Extract OSC dual duo filter Ha, Oiii and Sii channels"
+disabled = true # FIXME, debugging later stuff
+
+# FIXME-somehow-specify-what-filternames are used to auto detect this recipe can be used?
+# figure out how to support dual duo vs single duo.  Perhaps: the FIRST recipe that matches an auto rule
+# is used for any auto-defected defaults.  If an auto match is found it will be saved in the generated starter
+# project.toml file.
+
+# non OSC people use names like LRGB or SHO
+
+# for dual duo if we see Sii assume they also have HaOiii
+auto.for-filter = ["SiiOiii"]
+# for single duo look for this
+# auto.for-filter = ["HaOiii"]
+auto.for-camera = ["OSC"]
+
+tool = "siril"
+when = "session-light" # run once per session-config
+output = "FIXME"
+
+# FIXME, bias and flat should have been added to context by two previous stages.  But for now hardwire
+# Note: they should not have filename extensions (see strip_extension in the old process.py)
+context.bias = '/workspaces/starbash/images/masters/biases/2025-09-09_stacked.fits'
+
+context.sessionid = "2025-09-16" # FIXME, generate this by looping over all sessions (from outside this file)
+context.sessionconfig = "SiiOiii" # FIXME generate this by looping over all session configs
+context.light_base = "light_s{sessionid}_c{sessionconfig}"
+
+# FIXME until auto light finding is in
+input.source = "path"
+input.path = "/workspaces/starbash/images/from_astroboy/M 27/2025-09-16/LIGHT/*.fit*"
+
+script = '''
+    # Create a sequence from the raw light frames, seq file goes to process_dir
+    link {light_base} -out={process_dir}
+    cd {process_dir}
+
+    # Calibrate the light frames using master bias and flat
+    calibrate {light_base} -bias={bias} -flat=flat -cfa -equalize_cfa
+
+    # Remove background gradient on a per-frame basis (generates bkg_pp_light.seq)
+    seqsubsky pp_{light_base} 1
+
+    # FIXME only do this step for duo filters (refactor to share common light processing function)
+    seqextract_HaOIII bkg_pp_{light_base} -resample=ha
+    '''
+
+temporaries = ["FIXME"]
+
+[[stage]]
+
+# FIXME, eventually we could make it optional to even have a starbash.toml.  If we find an
+# starbash.py we could introspect it for a starbash_config dict.  And look inside that for description
+# "stage", "when" or whatever?
+
+description = "Stack OSC dual duo filter data, with separate Ha, Oiii and Sii channels"
+
+context.target = "M 27" # FIXME
+context.targets = "/workspaces/starbash/images/processed" # FIXME, do something smarter
+
+tool = "python"
+when = "session-stack" # run once after all session/session-config processing was done
+
+# if not specified starbash.py used
+# script-file = "script.py"
+
+# or inline python code instead of that function?
+#script = '''
+#    make_stacked("SiiOiii", "Ha", f"results_00001")
+#    ...
+#    (moved to script.py)
+#    '''

starbash/recipes/osc_single_duo/starbash.toml

@@ -0,0 +1,67 @@
+
+[repo]
+kind = "recipe"
+
+# all sb.toml files can optionally contain a version section.  if version of the running starbash app is out of bounds a warning message will be printed
+# to the user and the file will be ignored for future processing.
+[recipe.require.version]
+min="0.1.0"
+max="4.5.8"
+
+[recipe]
+author.name = "Kevin Hester"
+author.email = "kevinh@geeksville.com"
+
+[[stage]]
+
+description = "Process OSC single duo filter data, extracting Ha and Oiii"
+
+disabled = true # FIXME, we don't yet have auto selection based on filter types
+
+# Restrict processing of this stage to only if detected hardware was found for this session
+# for single duo look for this
+auto.for-filter = ["HaOiii"]
+auto.for-camera = ["OSC"]
+
+tool = "siril"
+when = "session-light" # run once per session-config
+output = "FIXME"
+
+script = '''
+    # Create a sequence from the raw light frames, seq file goes to process_dir
+    link {light_base} -out={process_dir}
+    cd {process_dir}
+
+    # Calibrate the light frames using master bias and flat
+    calibrate {light_base} -bias={bias} -flat={flat} -cfa -equalize_cfa
+
+    # Remove background gradient on a per-frame basis (generates bkg_pp_{light_base}.seq)
+    seqsubsky pp_{light_base} 1
+    '''
+
+temporaries = ["FIXME"]
+
+[[stage]]
+
+disabled = true # FIXME, we don't yet have auto selection based on filter types
+
+tool = "python"
+when = "session-stack" # run once after all session/session-config processing was done
+
+script-file = "script.py"
+
+# or inline python code instead of that function?
+script = '''
+    # green output channel - from the HaOiii filter Ha is on the 656nm red channel
+    make_stacked("HaOiii", "Ha", f"results_00001")
+
+    # blue output channel - both filters have Oiii on the 500nm blue channel.  Note the case here is uppercase to match siril output
+    make_stacked("*", "OIII", f"results_00002")
+
+    # There might be an old/state autogenerated .seq file, delete it so it doesn't confuse renormalize
+    results_seq_path = f"{process_dir}/results_.seq"
+    if os.path.exists(results_seq_path):
+        os.remove(results_seq_path)
+
+    make_renormalize()
+    '''

starbash/recipes/starbash.toml

@@ -0,0 +1,34 @@
+[repo]
+kind = "repo"
+
+# recipes for generating master flats, bias, darks...
+[[repo-ref]]
+dir = "master_bias"
+[[repo-ref]]
+dir = "master_flat"
+
+# Note: For automated recipe finding, it is important to list more demanding recipes first.  For instance:
+# osc_dual_duo (which needs a HaOiii and a SiiOiii filter) should be listed first, so that if we see
+# dataset containing both HaOiii and SiiOiii filters we will assume that is the recipe to use.
+#
+# But later if we see a dataset containing only HaOiii data, assume osc_single_duo should be used.
+
+[[repo-ref]]
+dir = "osc_dual_duo"
+
+[[repo-ref]]
+dir = "osc_single_duo"
+
+# processing stages, currently all declared here, but possibly in the future they could be added by user/other toml files
+
+[[stages]]
+name = "session-config" # for flat processing, master generation etc
+priority = 10
+
+[[stages]]
+name = "session-light" # generate light frames from lights and with reference to flats/bias
+priority = 20
+
+[[stages]]
+name = "session-stack" # stack frames
+priority = 30

starbash/repo/manager.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import logging
 from pathlib import Path
 from importlib import resources
+from typing import Any
 
 import tomlkit
 from tomlkit.toml_file import TOMLFile
@@ -42,29 +43,44 @@ class Repo:
 
     __repr__ = __str__
 
-    @property
-    def kind(self) -> str:
+    def kind(self, unknown_kind: str = "unknown") -> str:
         """
         Read-only attribute for the repository kind (e.g., "recipe", "data", etc.).
 
         Returns:
             The kind of the repository as a string.
         """
-        return str(self.get("repo.kind", "unknown"))
+        c = self.get("repo.kind", unknown_kind)
+        return str(c)
+
+    def add_repo_ref(self, dir: str) -> Repo | None:
+        """
+        Adds a new repo-ref to this repository's configuration.
+        if new returns the newly added Repo object, if already exists returns None"""
 
-    def add_repo_ref(self, dir: str) -> None:
+        # if dir is not absolute, we need to resolve it relative to the cwd
+        if not Path(dir).is_absolute():
+            dir = str((Path.cwd() / dir).resolve())
+
+        # Add the ref to this repo
         aot = self.config.get(REPO_REF, None)
         if aot is None:
             aot = tomlkit.aot()
-        else:
-            self.config.remove(
-                REPO_REF
-            )  # We want to completely replace it at the end of the file
+            self.config[REPO_REF] = aot  # add an empty AoT at the end of the file
+
+        if type(aot) is not AoT:
+            raise ValueError(f"repo-ref in {self.url} is not an array")
+
+        for t in aot:
+            if "dir" in t and t["dir"] == dir:
+                logging.warning(f"Repo ref {dir} already exists - ignoring.")
+                return None  # already exists
 
         ref = {"dir": dir}
         aot.append(ref)
-        self.config[REPO_REF] = aot
-        self.add_from_ref(ref)
+
+        # Also add the repo to the manager
+        return self.add_from_ref(ref)
 
     def write_config(self) -> None:
         """
@@ -107,25 +123,31 @@ class Repo:
 
         return None
 
-    def add_from_ref(self, ref: dict) -> None:
+    def add_from_ref(self, ref: dict) -> Repo:
         """
         Adds a repository based on a repo-ref dictionary.
         """
         if "url" in ref:
             url = ref["url"]
         elif "dir" in ref:
-            path = Path(ref["dir"])
-            base_path = self.get_path()
-            if base_path and not path.is_absolute():
-                # Resolve relative to the current TOML file's directory
-                path = (base_path / path).resolve()
+            # FIXME don't allow ~ or .. in file paths for security reasons?
+            if self.is_scheme("file"):
+                path = Path(ref["dir"])
+                base_path = self.get_path()
+
+                if base_path and not path.is_absolute():
+                    # Resolve relative to the current TOML file's directory
+                    path = (base_path / path).resolve()
+                else:
+                    # Expand ~ and resolve from CWD
+                    path = path.expanduser().resolve()
+                url = f"file://{path}"
             else:
-                # Expand ~ and resolve from CWD
-                path = path.expanduser().resolve()
-            url = f"file://{path}"
+                # construct an URL relative to this repo's URL
+                url = self.url.rstrip("/") + "/" + ref["dir"].lstrip("/")
         else:
             raise ValueError(f"Invalid repo reference: {ref}")
-        self.manager.add_repo(url)
+        return self.manager.add_repo(url)
 
     def add_by_repo_refs(self) -> None:
         """Add all repos mentioned by repo-refs in this repo's config."""
@@ -203,7 +225,7 @@ class Repo:
             )  # we currently make it optional to have the config file at root
             return tomlkit.TOMLDocument()  # empty placeholder
 
-    def get(self, key: str, default=None):
+    def get(self, key: str, default: Any | None = None) -> Any | None:
         """
         Gets a value from this repo's config for a given key.
         The key can be a dot-separated string for nested values.
@@ -222,6 +244,36 @@ class Repo:
             value = value.get(k)
         return value if value is not None else default
 
+    def set(self, key: str, value: Any) -> None:
+        """
+        Sets a value in this repo's config for a given key.
+        The key can be a dot-separated string for nested values.
+        Creates nested Table structures as needed.
+
+        Args:
+            key: The dot-separated key to set (e.g., "repo.kind").
+            value: The value to set.
+
+        Example:
+            repo.set("repo.kind", "preferences")
+            repo.set("user.name", "John Doe")
+        """
+        keys = key.split(".")
+        current: Any = self.config
+
+        # Navigate/create nested structure for all keys except the last
+        for k in keys[:-1]:
+            if k not in current:
+                # Create a new nested table
+                current[k] = tomlkit.table()
+            elif not isinstance(current[k], dict):
+                # Overwrite non-dict value with a table
+                current[k] = tomlkit.table()
+            current = current[k]
+
+        # Set the final value
+        current[keys[-1]] = value
+
 
 class RepoManager:
     """
@@ -245,6 +297,15 @@ class RepoManager:
         # Most users will just want to read from merged
         self.merged = MultiDict()
 
+    @property
+    def regular_repos(self) -> list[Repo]:
+        "We exclude certain repo types (preferences, recipe) from the list of repos users care about."
+        return [
+            r
+            for r in self.repos
+            if r.kind() not in ("preferences") and not r.is_scheme("pkg")
+        ]
+
     def add_repo(self, url: str) -> Repo:
         logging.debug(f"Adding repo: {url}")
         r = Repo(self, url)

starbash/selection.py

@@ -17,6 +17,7 @@ class Selection:
     - Date ranges
     - Filters
     - Image types
+    - Telescope names
 
     The selection state is saved to disk and can be used to build database queries.
     """
@@ -33,6 +34,7 @@ class Selection:
         self.date_end: Optional[str] = None
         self.filters: list[str] = []
         self.image_types: list[str] = []
+        self.telescopes: list[str] = []
 
         # Load existing state if it exists
         self._load()
@@ -48,6 +50,7 @@ class Selection:
                     self.date_end = data.get("date_end")
                     self.filters = data.get("filters", [])
                     self.image_types = data.get("image_types", [])
+                    self.telescopes = data.get("telescopes", [])
                 logging.debug(f"Loaded selection state from {self.state_file}")
             except Exception as e:
                 logging.warning(f"Failed to load selection state: {e}")
@@ -64,6 +67,7 @@ class Selection:
                 "date_end": self.date_end,
                 "filters": self.filters,
                 "image_types": self.image_types,
+                "telescopes": self.telescopes,
             }
 
             with open(self.state_file, "w") as f:
@@ -79,6 +83,7 @@ class Selection:
         self.date_end = None
         self.filters = []
         self.image_types = []
+        self.telescopes = []
         self._save()
 
     def add_target(self, target: str) -> None:
@@ -101,6 +106,26 @@ class Selection:
             self.targets.remove(target)
             self._save()
 
+    def add_telescope(self, telescope: str) -> None:
+        """Add a telescope to the selection.
+
+        Args:
+            telescope: Telescope name to add to the selection
+        """
+        if telescope not in self.telescopes:
+            self.telescopes.append(telescope)
+            self._save()
+
+    def remove_telescope(self, telescope: str) -> None:
+        """Remove a telescope from the selection.
+
+        Args:
+            telescope: Telescope name to remove from the selection
+        """
+        if telescope in self.telescopes:
+            self.telescopes.remove(telescope)
+            self._save()
+
     def set_date_range(
         self, start: Optional[str] = None, end: Optional[str] = None
     ) -> None:
@@ -146,6 +171,7 @@ class Selection:
             and self.date_end is None
             and not self.filters
             and not self.image_types
+            and not self.telescopes
         )
 
     def get_query_conditions(self) -> dict[str, Any]:
@@ -173,6 +199,13 @@ class Selection:
             # TODO: Support multiple filters in queries
             conditions["FILTER"] = self.filters[0] if len(self.filters) == 1 else None
 
+        if self.telescopes:
+            # For now, just use the first telescope
+            # TODO: Support multiple telescopes in queries
+            conditions["TELESCOP"] = (
+                self.telescopes[0] if len(self.telescopes) == 1 else None
+            )
+
         # Add date range conditions
         if self.date_start:
             conditions["date_start"] = self.date_start
@@ -198,6 +231,9 @@ class Selection:
         if self.targets:
             summary["criteria"].append(f"Targets: {', '.join(self.targets)}")
 
+        if self.telescopes:
+            summary["criteria"].append(f"Telescopes: {', '.join(self.telescopes)}")
+
         if self.date_start or self.date_end:
             date_range = []
             if self.date_start: