starbash 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl

starbash/database.py

@@ -5,9 +5,13 @@ from pathlib import Path
 from typing import Any, Optional
 from datetime import datetime, timedelta
 import json
+from typing import TypeAlias
 
 from .paths import get_user_data_dir
 
+SessionRow: TypeAlias = dict[str, Any]
+ImageRow: TypeAlias = dict[str, Any]
+
 
 def get_column_name(k: str) -> str:
     """Convert keynames to SQL legal column names"""
@@ -21,10 +25,35 @@ class Database:
     """SQLite-backed application database.
 
     Stores data under the OS-specific user data directory using platformdirs.
+
+    Tables:
+    #1: repos
+    A table with one row per repository. Contains only 'id' (primary key) and 'url' (unique).
+    The URL identifies the repository root (e.g., 'file:///path/to/repo').
+
+    #2: images
     Provides an `images` table for FITS metadata and basic helpers.
 
     The images table stores DATE-OBS and DATE as indexed SQL columns for
     efficient date-based queries, while other FITS metadata is stored in JSON.
+
+    The 'path' column contains a path **relative** to the repository root.
+    Each image belongs to exactly one repo, linked via the repo_id foreign key.
+    The combination of (repo_id, path) is unique.
+
+    Image retrieval methods (get_image, search_image, all_images) join with the repos
+    table to include repo_url in results, allowing callers to reconstruct absolute paths.
+
+    #3: sessions
+    The sessions table has one row per observing session, summarizing key info.
+    Sessions are identified by filter, image type, target, telescope, etc, and start/end times.
+    They correspond to groups of images taken together during an observing run (e.g.
+    session start/end describes the range of images DATE-OBS).
+
+    Each session also has an image_doc_id field which will point to a representative
+    image in the images table. Eventually we'll use joins to add extra info from images to
+    the exposed 'session' row.
+
     """
 
     EXPTIME_KEY = "EXPTIME"
@@ -35,13 +64,15 @@ class Database:
     EXPTIME_TOTAL_KEY = "exptime-total"
     DATE_OBS_KEY = "DATE-OBS"
     DATE_KEY = "DATE"
-    IMAGE_DOC_KEY = "image-doc"
+    IMAGE_DOC_KEY = "image-doc-id"
     IMAGETYP_KEY = "IMAGETYP"
     OBJECT_KEY = "OBJECT"
     TELESCOP_KEY = "TELESCOP"
+    ID_KEY = "id"  # for finding any row by its ID
 
     SESSIONS_TABLE = "sessions"
     IMAGES_TABLE = "images"
+    REPOS_TABLE = "repos"
 
     def __init__(
         self,
@@ -64,18 +95,38 @@ class Database:
         self._init_tables()
 
     def _init_tables(self) -> None:
-        """Create the images and sessions tables if they don't exist."""
+        """Create the repos, images and sessions tables if they don't exist."""
         cursor = self._db.cursor()
 
+        # Create repos table
+        cursor.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS {self.REPOS_TABLE} (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                url TEXT UNIQUE NOT NULL
+            )
+        """
+        )
+
+        # Create index on url for faster lookups
+        cursor.execute(
+            f"""
+            CREATE INDEX IF NOT EXISTS idx_repos_url ON {self.REPOS_TABLE}(url)
+        """
+        )
+
         # Create images table with DATE-OBS and DATE as indexed columns
         cursor.execute(
             f"""
             CREATE TABLE IF NOT EXISTS {self.IMAGES_TABLE} (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
-                path TEXT UNIQUE NOT NULL,
+                repo_id INTEGER NOT NULL,
+                path TEXT NOT NULL,
                 date_obs TEXT,
                 date TEXT,
-                metadata TEXT NOT NULL
+                metadata TEXT NOT NULL,
+                FOREIGN KEY (repo_id) REFERENCES {self.REPOS_TABLE}(id),
+                UNIQUE(repo_id, path)
             )
         """
         )
@@ -114,7 +165,8 @@ class Database:
                 telescop TEXT NOT NULL,
                 num_images INTEGER NOT NULL,
                 exptime_total REAL NOT NULL,
-                image_doc_id INTEGER
+                image_doc_id INTEGER,
+                FOREIGN KEY (image_doc_id) REFERENCES {self.IMAGES_TABLE}(id)
             )
         """
         )
@@ -129,18 +181,125 @@ class Database:
 
         self._db.commit()
 
+    # --- Convenience helpers for common repo operations ---
+    def remove_repo(self, url: str) -> None:
+        """Remove a repo record by URL.
+
+        This will cascade delete all images belonging to this repo, and all sessions
+        that reference those images.
+
+        Args:
+            url: The repository URL (e.g., 'file:///path/to/repo')
+        """
+        cursor = self._db.cursor()
+
+        # First get the repo_id
+        repo_id = self.get_repo_id(url)
+        if repo_id is None:
+            return  # Repo doesn't exist, nothing to delete
+
+        # Delete sessions that reference images from this repo
+        # This deletes sessions where image_doc_id points to any image in this repo
+        cursor.execute(
+            f"""
+            DELETE FROM {self.SESSIONS_TABLE}
+            WHERE image_doc_id IN (
+                SELECT id FROM {self.IMAGES_TABLE} WHERE repo_id = ?
+            )
+            """,
+            (repo_id,),
+        )
+
+        # Delete all images from this repo
+        cursor.execute(
+            f"DELETE FROM {self.IMAGES_TABLE} WHERE repo_id = ?",
+            (repo_id,),
+        )
+
+        # Finally delete the repo itself
+        cursor.execute(f"DELETE FROM {self.REPOS_TABLE} WHERE id = ?", (repo_id,))
+
+        self._db.commit()
+
+    def upsert_repo(self, url: str) -> int:
+        """Insert or update a repo record by unique URL.
+
+        Args:
+            url: The repository URL (e.g., 'file:///path/to/repo')
+
+        Returns:
+            The rowid of the inserted/updated record.
+        """
+        cursor = self._db.cursor()
+        cursor.execute(
+            f"""
+            INSERT INTO {self.REPOS_TABLE} (url) VALUES (?)
+            ON CONFLICT(url) DO NOTHING
+        """,
+            (url,),
+        )
+
+        self._db.commit()
+
+        # Get the rowid of the inserted/existing record
+        cursor.execute(f"SELECT id FROM {self.REPOS_TABLE} WHERE url = ?", (url,))
+        result = cursor.fetchone()
+        if result:
+            return result[0]
+        return cursor.lastrowid if cursor.lastrowid is not None else 0
+
+    def get_repo_id(self, url: str) -> int | None:
+        """Get the repo_id for a given URL.
+
+        Args:
+            url: The repository URL
+
+        Returns:
+            The repo_id if found, None otherwise
+        """
+        cursor = self._db.cursor()
+        cursor.execute(f"SELECT id FROM {self.REPOS_TABLE} WHERE url = ?", (url,))
+        result = cursor.fetchone()
+        return result[0] if result else None
+
+    def get_repo_url(self, repo_id: int) -> str | None:
+        """Get the URL for a given repo_id.
+
+        Args:
+            repo_id: The repository ID
+
+        Returns:
+            The URL if found, None otherwise
+        """
+        cursor = self._db.cursor()
+        cursor.execute(f"SELECT url FROM {self.REPOS_TABLE} WHERE id = ?", (repo_id,))
+        result = cursor.fetchone()
+        return result[0] if result else None
+
     # --- Convenience helpers for common image operations ---
-    def upsert_image(self, record: dict[str, Any]) -> int:
+    def upsert_image(self, record: dict[str, Any], repo_url: str) -> int:
         """Insert or update an image record by unique path.
 
-        The record must include a 'path' key; other keys are arbitrary FITS metadata.
+        The record must include a 'path' key (relative to repo); other keys are arbitrary FITS metadata.
+        The path is stored as-is - caller is responsible for making it relative to the repo.
         DATE-OBS and DATE are extracted and stored as indexed columns for efficient queries.
-        Returns the rowid of the inserted/updated record.
+
+        Args:
+            record: Dictionary containing image metadata including 'path' (relative to repo)
+            repo_url: The repository URL this image belongs to
+
+        Returns:
+            The rowid of the inserted/updated record.
         """
         path = record.get("path")
         if not path:
             raise ValueError("record must include 'path'")
 
+        # Get or create the repo_id for this URL
+        repo_id = self.get_repo_id(repo_url)
+        if repo_id is None:
+            repo_id = self.upsert_repo(repo_url)
+
         # Extract date fields for column storage
         date_obs = record.get(self.DATE_OBS_KEY)
         date = record.get(self.DATE_KEY)
@@ -152,25 +311,28 @@ class Database:
         cursor = self._db.cursor()
         cursor.execute(
             f"""
-            INSERT INTO {self.IMAGES_TABLE} (path, date_obs, date, metadata) VALUES (?, ?, ?, ?)
-            ON CONFLICT(path) DO UPDATE SET
+            INSERT INTO {self.IMAGES_TABLE} (repo_id, path, date_obs, date, metadata) VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(repo_id, path) DO UPDATE SET
                 date_obs = excluded.date_obs,
                 date = excluded.date,
                 metadata = excluded.metadata
         """,
-            (path, date_obs, date, metadata_json),
+            (repo_id, str(path), date_obs, date, metadata_json),
         )
 
         self._db.commit()
 
         # Get the rowid of the inserted/updated record
-        cursor.execute(f"SELECT id FROM {self.IMAGES_TABLE} WHERE path = ?", (path,))
+        cursor.execute(
+            f"SELECT id FROM {self.IMAGES_TABLE} WHERE repo_id = ? AND path = ?",
+            (repo_id, str(path)),
+        )
         result = cursor.fetchone()
         if result:
             return result[0]
         return cursor.lastrowid if cursor.lastrowid is not None else 0
 
-    def search_image(self, conditions: dict[str, Any]) -> list[dict[str, Any]] | None:
+    def search_image(self, conditions: dict[str, Any]) -> list[SessionRow]:
         """Search for images matching the given conditions.
 
         Args:
@@ -180,7 +342,7 @@ class Database:
                        - 'date_end': Filter images with DATE-OBS <= this date
 
         Returns:
-            List of matching image records or None if no matches
+            List of matching image records with relative path, repo_id, and repo_url
         """
         # Extract special date filter keys (make a copy to avoid modifying caller's dict)
         conditions_copy = dict(conditions)
@@ -192,15 +354,19 @@ class Database:
         params = []
 
         if date_start:
-            where_clauses.append("date_obs >= ?")
+            where_clauses.append("i.date_obs >= ?")
             params.append(date_start)
 
         if date_end:
-            where_clauses.append("date_obs <= ?")
+            where_clauses.append("i.date_obs <= ?")
             params.append(date_end)
 
-        # Build the query
-        query = f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE}"
+        # Build the query with JOIN to repos table
+        query = f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+        """
         if where_clauses:
             query += " WHERE " + " AND ".join(where_clauses)
 
@@ -210,7 +376,10 @@ class Database:
         results = []
         for row in cursor.fetchall():
             metadata = json.loads(row["metadata"])
+            # Store the relative path, repo_id, and repo_url for caller
             metadata["path"] = row["path"]
+            metadata["repo_id"] = row["repo_id"]
+            metadata["repo_url"] = row["repo_url"]
             metadata["id"] = row["id"]
 
             # Add date fields back to metadata for compatibility
@@ -225,58 +394,11 @@ class Database:
             if match:
                 results.append(metadata)
 
-        return results if results else None
-
-    def where_session(self, conditions: dict[str, Any] | None) -> tuple[str, list[Any]]:
-        """Search for sessions matching the given conditions.
-
-        Args:
-            conditions: Dictionary of session key-value pairs to match, or None for all.
-                       Special keys:
-                       - 'date_start': Filter sessions starting on or after this date
-                       - 'date_end': Filter sessions starting on or before this date
-
-        Returns:
-            Tuple of (WHERE clause string, list of parameters)
-        """
-        if conditions is None:
-            conditions = {}
-
-        # Build WHERE clause dynamically based on conditions
-        where_clauses = []
-        params = []
-
-        # Extract date range conditions
-        date_start = conditions.get("date_start")
-        date_end = conditions.get("date_end")
-
-        # Add date range filters to WHERE clause
-        if date_start:
-            where_clauses.append("start >= ?")
-            params.append(date_start)
-
-        if date_end:
-            where_clauses.append("start <= ?")
-            params.append(date_end)
-
-        # Add standard conditions to WHERE clause
-        for key, value in conditions.items():
-            if key not in ("date_start", "date_end") and value is not None:
-                column_name = key
-                where_clauses.append(f"{column_name} = ?")
-                params.append(value)
-
-        # Build the query
-        query = ""
-
-        if where_clauses:
-            query += " WHERE " + " AND ".join(where_clauses)
-
-        return (query, params)
+        return results
 
     def search_session(
-        self, conditions: dict[str, Any] | None = None
-    ) -> list[dict[str, Any]]:
+        self, where_tuple: tuple[str, list[Any]] = ("", [])
+    ) -> list[SessionRow]:
         """Search for sessions matching the given conditions.
 
         Args:
@@ -286,26 +408,31 @@ class Database:
                        - 'date_end': Filter sessions starting on or before this date
 
         Returns:
-            List of matching session records or None
+            List of matching session records with metadata from the reference image
         """
-        if conditions is None:
-            conditions = {}
-
         # Build WHERE clause dynamically based on conditions
-        where_clause, params = self.where_session(conditions)
+        where_clause, params = where_tuple
 
-        # Build the query
+        # Build the query with JOIN to images table to get reference image metadata
         query = f"""
-            SELECT id, start, end, filter, imagetyp, object, telescop,
-                   num_images, exptime_total, image_doc_id
-            FROM {self.SESSIONS_TABLE}
+            SELECT s.id, s.start, s.end, s.filter, s.imagetyp, s.object, s.telescop,
+                   s.num_images, s.exptime_total, s.image_doc_id, i.metadata
+            FROM {self.SESSIONS_TABLE} s
+            LEFT JOIN {self.IMAGES_TABLE} i ON s.image_doc_id = i.id
             {where_clause}
         """
 
         cursor = self._db.cursor()
         cursor.execute(query, params)
 
-        results = [dict(row) for row in cursor.fetchall()]
+        results = []
+        for row in cursor.fetchall():
+            session_dict = dict(row)
+            # Parse the metadata JSON if it exists
+            if session_dict.get("metadata"):
+                session_dict["metadata"] = json.loads(session_dict["metadata"])
+            results.append(session_dict)
+
         return results
 
     def len_table(self, table_name: str) -> int:
@@ -333,20 +460,35 @@ class Database:
         result = cursor.fetchone()
         return result[0] if result and result[0] is not None else 0
 
-    def get_image(self, path: str) -> dict[str, Any] | None:
-        """Get an image record by path."""
+    def get_image(self, repo_url: str, path: str) -> ImageRow | None:
+        """Get an image record by repo_url and relative path.
+
+        Args:
+            repo_url: The repository URL
+            path: Path relative to the repository root
+
+        Returns:
+            Image record with relative path, repo_id, and repo_url, or None if not found
+        """
         cursor = self._db.cursor()
         cursor.execute(
-            f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE} WHERE path = ?",
-            (path,),
+            f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+            WHERE r.url = ? AND i.path = ?
+            """,
+            (repo_url, path),
         )
-        row = cursor.fetchone()
 
+        row = cursor.fetchone()
         if row is None:
             return None
 
         metadata = json.loads(row["metadata"])
         metadata["path"] = row["path"]
+        metadata["repo_id"] = row["repo_id"]
+        metadata["repo_url"] = row["repo_url"]
         metadata["id"] = row["id"]
 
         # Add date fields back to metadata for compatibility
@@ -357,17 +499,24 @@ class Database:
 
         return metadata
 
-    def all_images(self) -> list[dict[str, Any]]:
-        """Return all image records."""
+    def all_images(self) -> list[ImageRow]:
+        """Return all image records with relative paths, repo_id, and repo_url."""
         cursor = self._db.cursor()
         cursor.execute(
-            f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE}"
+            f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+            """
         )
 
         results = []
         for row in cursor.fetchall():
             metadata = json.loads(row["metadata"])
+            # Return relative path, repo_id, and repo_url for caller
             metadata["path"] = row["path"]
+            metadata["repo_id"] = row["repo_id"]
+            metadata["repo_url"] = row["repo_url"]
             metadata["id"] = row["id"]
 
             # Add date fields back to metadata for compatibility
@@ -406,7 +555,7 @@ class Database:
 
         return dict(row)
 
-    def get_session(self, to_find: dict[str, str]) -> dict[str, Any] | None:
+    def get_session(self, to_find: dict[str, str]) -> SessionRow | None:
         """Find a session matching the given criteria.
 
         Searches for sessions with the same filter, image type, target, and telescope
@@ -452,7 +601,7 @@ class Database:
         return dict(row)
 
     def upsert_session(
-        self, new: dict[str, Any], existing: dict[str, Any] | None = None
+        self, new: SessionRow, existing: SessionRow | None = None
     ) -> None:
         """Insert or update a session record."""
         cursor = self._db.cursor()

starbash/defaults/starbash.toml

@@ -3,6 +3,23 @@
 [repo]
 kind = "preferences"
 
+[aliases]
+# aliases can be used to map non standard (or non english) frame names to standard terms
+# This is also used to map filters based on common misspellings or variations.
+# We assume the first listed option in the list is the 'canonical' name used for printing etc...
+
+# frame types
+dark = ["dark", "darks"]
+flat = ["flat", "flats"]
+bias = ["bias", "biases"]
+
+# file suffixes
+fit = ["fits", "fit"]
+
+# filter names
+SiiOiii = ["SiiOiii", "SII-OIII", "S2-O3"]
+HaOiii = ["HaOiii", "HA-OIII", "Halpha-O3"]
+
 # FIXME, somewhere here list default patterns which can be used to identify NINA, ASIAIR, SEESTAR
 # raw repo layouts
 

starbash/main.py

@@ -6,7 +6,7 @@ import starbash.url as url
 import starbash
 
 from .app import Starbash, get_user_config_path, setup_logging
-from .commands import info, repo, select, user
+from .commands import info, process, repo, select, user
 from . import console
 
 app = typer.Typer(
@@ -17,6 +17,9 @@ app.add_typer(user.app, name="user", help="Manage user settings.")
 app.add_typer(repo.app, name="repo", help="Manage Starbash repositories.")
 app.add_typer(select.app, name="select", help="Manage session and target selection.")
 app.add_typer(info.app, name="info", help="Display system and data information.")
+app.add_typer(
+    process.app, name="process", help="Process images using automated workflows."
+)
 
 
 @app.callback(invoke_without_command=True)

starbash/recipes/master_bias/starbash.toml

@@ -10,7 +10,7 @@ author.email = "FIXMESiril?"
 [[stage]]
 
 description = "Generate master bias"
-disabled = true # FIXME, debugging later stuff
+disabled = false # FIXME, debugging later stuff
 
 # Restrict processing of this stage to only if detected hardware was found for this session
 # For any camera
@@ -23,9 +23,9 @@ tool = "siril"
 # input.source = "most-recent" # only look for the most recent set of raws for this particular type
 input.type = "bias" # look in all raw repos, but look only for bias files
 
-# for early development we have support for simple absolute file paths with globs
-input.source = "path"
-input.path = "/workspaces/starbash/images/from_astroboy/masters-raw/2025-09-09/BIAS/*.fit*"
+# Look for files in input repos, finding them by using the "relative" tag they contain
+input.source = "repo"
+# input.path = ".../from_astroboy/masters-raw/2025-09-09/BIAS/*.fit*"
 input.required = true # Is at least one input file required?  If true, we will skip running this stage entirely (with a warning)
 
 # make the following also work
@@ -34,7 +34,17 @@ input.required = true # Is at least one input file required?  If true, we will s
 #os.makedirs(os.path.dirname(process_dir), exist_ok=True)
 #frames = glob(f"{masters_raw}/{date}/BIAS/{date}_*.fit*")
 #siril_run_in_temp_dir(frames, ...
-when = "session-config" # run at the start of each session process
+when = "setup.masters" # run when master biases are regenerated
+
+# Based on the following definitions in the stage toml file...
+output.dest = "repo" # write to a particular repo
+output.type = "master" # write output to the special masters repo
+
+# the following fields will be auto populated in the context before entry:
+# context.output.base_path - the full filepath to write the output file to **excluding the suffix**
+# context.output.full_path - the full filepath to write the output file to (including suffix)
+# (NOT implemented / needed) context.output.root_path - points to the base of the destination repo
+# (NOT implemented / needed) context.output.suffix - the suffix to append to the output file (e.g. .fits or .fit.gz)
 
 # The following constants are auto defined before running the tool
 # context.process_dir (points to the session specific semi-persistent local dir for that sessions written/read data files)
@@ -42,8 +52,15 @@ when = "session-config" # run at the start of each session process
 # context.temp_dir (points to a temporary directory this tool can use for writing)
 
 # Everything in the constants dict will be predefined as named variables for use by the script
-context.date = "2025-09-09"  # FIXME - later find auto latest date with bias frames
-context.output = "{masters}/biases/{date}_stacked.fits" # if the output already exists processing will be skipped
+# context.date = "2025-09-0.9"  # FIXME - later find auto latest date with bias frames
+
+# output file will be place in the masters repo
+# if the output already exists processing will be skipped
+#
+# with a path like "{instrument}/{date}/{imagetyp}/{sessionconfig}.fits"
+# this path comes from master_repo.relative
+# context.output = "{output.root}/{instrument}/{date}/{imagetyp}/{sessionconfig}.fits"
+
 
 script = '''
     # Convert Bias Frames to .fit files
@@ -51,5 +68,5 @@ script = '''
     cd {process_dir}
 
     # Stack Bias Frames to bias_stacked.fit
-    stack bias rej 3 3 -nonorm -out={output}
+    stack bias rej 3 3 -nonorm -out={output["base_path"]}
     '''

starbash/recipes/starbash.toml

@@ -21,6 +21,11 @@ dir = "osc_single_duo"
 
 # processing stages, currently all declared here, but possibly in the future they could be added by user/other toml files
 
+# Not included in standard list - for now we run manually
+#[[stages]]
+#name = "setup-masters" # for flat processing, master generation etc
+#priority = 5
+
 [[stages]]
 name = "session-config" # for flat processing, master generation etc
 priority = 10