starbash 0.1.6__py3-none-any.whl → 0.1.9__py3-none-any.whl

starbash/database.py

@@ -1,23 +1,59 @@
 from __future__ import annotations
 
-import logging
 import sqlite3
 from pathlib import Path
 from typing import Any, Optional
 from datetime import datetime, timedelta
 import json
+from typing import TypeAlias
 
 from .paths import get_user_data_dir
 
+SessionRow: TypeAlias = dict[str, Any]
+ImageRow: TypeAlias = dict[str, Any]
+
+
+def get_column_name(k: str) -> str:
+    """Convert keynames to SQL legal column names"""
+    k = k.lower()
+    k = k.replace(" ", "_")
+    k = k.replace("-", "_")
+    return k
+
 
 class Database:
     """SQLite-backed application database.
 
     Stores data under the OS-specific user data directory using platformdirs.
+
+    Tables:
+    #1: repos
+    A table with one row per repository. Contains only 'id' (primary key) and 'url' (unique).
+    The URL identifies the repository root (e.g., 'file:///path/to/repo').
+
+    #2: images
     Provides an `images` table for FITS metadata and basic helpers.
 
     The images table stores DATE-OBS and DATE as indexed SQL columns for
     efficient date-based queries, while other FITS metadata is stored in JSON.
+
+    The 'path' column contains a path **relative** to the repository root.
+    Each image belongs to exactly one repo, linked via the repo_id foreign key.
+    The combination of (repo_id, path) is unique.
+
+    Image retrieval methods (get_image, search_image, all_images) join with the repos
+    table to include repo_url in results, allowing callers to reconstruct absolute paths.
+
+    #3: sessions
+    The sessions table has one row per observing session, summarizing key info.
+    Sessions are identified by filter, image type, target, telescope, etc, and start/end times.
+    They correspond to groups of images taken together during an observing run (e.g.
+    session start/end describes the range of images DATE-OBS).
+
+    Each session also has an image_doc_id field which will point to a representative
+    image in the images table. Eventually we'll use joins to add extra info from images to
+    the exposed 'session' row.
+
     """
 
     EXPTIME_KEY = "EXPTIME"
@@ -28,13 +64,15 @@ class Database:
     EXPTIME_TOTAL_KEY = "exptime-total"
     DATE_OBS_KEY = "DATE-OBS"
     DATE_KEY = "DATE"
-    IMAGE_DOC_KEY = "image-doc"
+    IMAGE_DOC_KEY = "image-doc-id"
     IMAGETYP_KEY = "IMAGETYP"
     OBJECT_KEY = "OBJECT"
     TELESCOP_KEY = "TELESCOP"
+    ID_KEY = "id"  # for finding any row by its ID
 
     SESSIONS_TABLE = "sessions"
     IMAGES_TABLE = "images"
+    REPOS_TABLE = "repos"
 
     def __init__(
         self,
@@ -57,18 +95,38 @@ class Database:
         self._init_tables()
 
     def _init_tables(self) -> None:
-        """Create the images and sessions tables if they don't exist."""
+        """Create the repos, images and sessions tables if they don't exist."""
         cursor = self._db.cursor()
 
+        # Create repos table
+        cursor.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS {self.REPOS_TABLE} (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                url TEXT UNIQUE NOT NULL
+            )
+        """
+        )
+
+        # Create index on url for faster lookups
+        cursor.execute(
+            f"""
+            CREATE INDEX IF NOT EXISTS idx_repos_url ON {self.REPOS_TABLE}(url)
+        """
+        )
+
         # Create images table with DATE-OBS and DATE as indexed columns
         cursor.execute(
             f"""
             CREATE TABLE IF NOT EXISTS {self.IMAGES_TABLE} (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
-                path TEXT UNIQUE NOT NULL,
+                repo_id INTEGER NOT NULL,
+                path TEXT NOT NULL,
                 date_obs TEXT,
                 date TEXT,
-                metadata TEXT NOT NULL
+                metadata TEXT NOT NULL,
+                FOREIGN KEY (repo_id) REFERENCES {self.REPOS_TABLE}(id),
+                UNIQUE(repo_id, path)
             )
         """
         )
@@ -107,7 +165,8 @@ class Database:
                 telescop TEXT NOT NULL,
                 num_images INTEGER NOT NULL,
                 exptime_total REAL NOT NULL,
-                image_doc_id INTEGER
+                image_doc_id INTEGER,
+                FOREIGN KEY (image_doc_id) REFERENCES {self.IMAGES_TABLE}(id)
             )
         """
         )
@@ -122,18 +181,125 @@ class Database:
 
         self._db.commit()
 
+    # --- Convenience helpers for common repo operations ---
+    def remove_repo(self, url: str) -> None:
+        """Remove a repo record by URL.
+
+        This will cascade delete all images belonging to this repo, and all sessions
+        that reference those images.
+
+        Args:
+            url: The repository URL (e.g., 'file:///path/to/repo')
+        """
+        cursor = self._db.cursor()
+
+        # First get the repo_id
+        repo_id = self.get_repo_id(url)
+        if repo_id is None:
+            return  # Repo doesn't exist, nothing to delete
+
+        # Delete sessions that reference images from this repo
+        # This deletes sessions where image_doc_id points to any image in this repo
+        cursor.execute(
+            f"""
+            DELETE FROM {self.SESSIONS_TABLE}
+            WHERE image_doc_id IN (
+                SELECT id FROM {self.IMAGES_TABLE} WHERE repo_id = ?
+            )
+            """,
+            (repo_id,),
+        )
+
+        # Delete all images from this repo
+        cursor.execute(
+            f"DELETE FROM {self.IMAGES_TABLE} WHERE repo_id = ?",
+            (repo_id,),
+        )
+
+        # Finally delete the repo itself
+        cursor.execute(f"DELETE FROM {self.REPOS_TABLE} WHERE id = ?", (repo_id,))
+
+        self._db.commit()
+
+    def upsert_repo(self, url: str) -> int:
+        """Insert or update a repo record by unique URL.
+
+        Args:
+            url: The repository URL (e.g., 'file:///path/to/repo')
+
+        Returns:
+            The rowid of the inserted/updated record.
+        """
+        cursor = self._db.cursor()
+        cursor.execute(
+            f"""
+            INSERT INTO {self.REPOS_TABLE} (url) VALUES (?)
+            ON CONFLICT(url) DO NOTHING
+        """,
+            (url,),
+        )
+
+        self._db.commit()
+
+        # Get the rowid of the inserted/existing record
+        cursor.execute(f"SELECT id FROM {self.REPOS_TABLE} WHERE url = ?", (url,))
+        result = cursor.fetchone()
+        if result:
+            return result[0]
+        return cursor.lastrowid if cursor.lastrowid is not None else 0
+
+    def get_repo_id(self, url: str) -> int | None:
+        """Get the repo_id for a given URL.
+
+        Args:
+            url: The repository URL
+
+        Returns:
+            The repo_id if found, None otherwise
+        """
+        cursor = self._db.cursor()
+        cursor.execute(f"SELECT id FROM {self.REPOS_TABLE} WHERE url = ?", (url,))
+        result = cursor.fetchone()
+        return result[0] if result else None
+
+    def get_repo_url(self, repo_id: int) -> str | None:
+        """Get the URL for a given repo_id.
+
+        Args:
+            repo_id: The repository ID
+
+        Returns:
+            The URL if found, None otherwise
+        """
+        cursor = self._db.cursor()
+        cursor.execute(f"SELECT url FROM {self.REPOS_TABLE} WHERE id = ?", (repo_id,))
+        result = cursor.fetchone()
+        return result[0] if result else None
+
     # --- Convenience helpers for common image operations ---
-    def upsert_image(self, record: dict[str, Any]) -> int:
+    def upsert_image(self, record: dict[str, Any], repo_url: str) -> int:
         """Insert or update an image record by unique path.
 
-        The record must include a 'path' key; other keys are arbitrary FITS metadata.
+        The record must include a 'path' key (relative to repo); other keys are arbitrary FITS metadata.
+        The path is stored as-is - caller is responsible for making it relative to the repo.
         DATE-OBS and DATE are extracted and stored as indexed columns for efficient queries.
-        Returns the rowid of the inserted/updated record.
+
+        Args:
+            record: Dictionary containing image metadata including 'path' (relative to repo)
+            repo_url: The repository URL this image belongs to
+
+        Returns:
+            The rowid of the inserted/updated record.
         """
         path = record.get("path")
         if not path:
             raise ValueError("record must include 'path'")
 
+        # Get or create the repo_id for this URL
+        repo_id = self.get_repo_id(repo_url)
+        if repo_id is None:
+            repo_id = self.upsert_repo(repo_url)
+
         # Extract date fields for column storage
         date_obs = record.get(self.DATE_OBS_KEY)
         date = record.get(self.DATE_KEY)
@@ -145,25 +311,28 @@ class Database:
         cursor = self._db.cursor()
         cursor.execute(
             f"""
-            INSERT INTO {self.IMAGES_TABLE} (path, date_obs, date, metadata) VALUES (?, ?, ?, ?)
-            ON CONFLICT(path) DO UPDATE SET
+            INSERT INTO {self.IMAGES_TABLE} (repo_id, path, date_obs, date, metadata) VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(repo_id, path) DO UPDATE SET
                 date_obs = excluded.date_obs,
                 date = excluded.date,
                 metadata = excluded.metadata
         """,
-            (path, date_obs, date, metadata_json),
+            (repo_id, str(path), date_obs, date, metadata_json),
         )
 
         self._db.commit()
 
         # Get the rowid of the inserted/updated record
-        cursor.execute(f"SELECT id FROM {self.IMAGES_TABLE} WHERE path = ?", (path,))
+        cursor.execute(
+            f"SELECT id FROM {self.IMAGES_TABLE} WHERE repo_id = ? AND path = ?",
+            (repo_id, str(path)),
+        )
         result = cursor.fetchone()
         if result:
             return result[0]
         return cursor.lastrowid if cursor.lastrowid is not None else 0
 
-    def search_image(self, conditions: dict[str, Any]) -> list[dict[str, Any]] | None:
+    def search_image(self, conditions: dict[str, Any]) -> list[SessionRow]:
         """Search for images matching the given conditions.
 
         Args:
@@ -173,7 +342,7 @@ class Database:
                        - 'date_end': Filter images with DATE-OBS <= this date
 
         Returns:
-            List of matching image records or None if no matches
+            List of matching image records with relative path, repo_id, and repo_url
         """
         # Extract special date filter keys (make a copy to avoid modifying caller's dict)
         conditions_copy = dict(conditions)
@@ -185,15 +354,19 @@ class Database:
         params = []
 
         if date_start:
-            where_clauses.append("date_obs >= ?")
+            where_clauses.append("i.date_obs >= ?")
             params.append(date_start)
 
         if date_end:
-            where_clauses.append("date_obs <= ?")
+            where_clauses.append("i.date_obs <= ?")
             params.append(date_end)
 
-        # Build the query
-        query = f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE}"
+        # Build the query with JOIN to repos table
+        query = f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+        """
         if where_clauses:
             query += " WHERE " + " AND ".join(where_clauses)
 
@@ -203,7 +376,10 @@ class Database:
         results = []
         for row in cursor.fetchall():
             metadata = json.loads(row["metadata"])
+            # Store the relative path, repo_id, and repo_url for caller
             metadata["path"] = row["path"]
+            metadata["repo_id"] = row["repo_id"]
+            metadata["repo_url"] = row["repo_url"]
             metadata["id"] = row["id"]
 
             # Add date fields back to metadata for compatibility
@@ -218,11 +394,11 @@ class Database:
             if match:
                 results.append(metadata)
 
-        return results if results else None
+        return results
 
     def search_session(
-        self, conditions: dict[str, Any] | None
-    ) -> list[dict[str, Any]] | None:
+        self, where_tuple: tuple[str, list[Any]] = ("", [])
+    ) -> list[SessionRow]:
         """Search for sessions matching the given conditions.
 
         Args:
@@ -232,62 +408,32 @@ class Database:
                        - 'date_end': Filter sessions starting on or before this date
 
         Returns:
-            List of matching session records or None
+            List of matching session records with metadata from the reference image
         """
-        if conditions is None:
-            return self.all_sessions()
-
-        cursor = self._db.cursor()
-        cursor.execute(
-            f"""
-            SELECT id, start, end, filter, imagetyp, object, telescop,
-                   num_images, exptime_total, image_doc_id
-            FROM {self.SESSIONS_TABLE}
+        # Build WHERE clause dynamically based on conditions
+        where_clause, params = where_tuple
+
+        # Build the query with JOIN to images table to get reference image metadata
+        query = f"""
+            SELECT s.id, s.start, s.end, s.filter, s.imagetyp, s.object, s.telescop,
+                   s.num_images, s.exptime_total, s.image_doc_id, i.metadata
+            FROM {self.SESSIONS_TABLE} s
+            LEFT JOIN {self.IMAGES_TABLE} i ON s.image_doc_id = i.id
+            {where_clause}
         """
-        )
-
-        # Extract date range conditions if present
-        date_start = conditions.get("date_start")
-        date_end = conditions.get("date_end")
 
-        # Create a copy without date range keys for standard matching
-        standard_conditions = {
-            k: v
-            for k, v in conditions.items()
-            if k not in ("date_start", "date_end") and v is not None
-        }
+        cursor = self._db.cursor()
+        cursor.execute(query, params)
 
         results = []
         for row in cursor.fetchall():
-            session = {
-                "id": row["id"],
-                self.START_KEY: row["start"],
-                self.END_KEY: row["end"],
-                self.FILTER_KEY: row["filter"],
-                self.IMAGETYP_KEY: row["imagetyp"],
-                self.OBJECT_KEY: row["object"],
-                self.TELESCOP_KEY: row["telescop"],
-                self.NUM_IMAGES_KEY: row["num_images"],
-                self.EXPTIME_TOTAL_KEY: row["exptime_total"],
-                self.IMAGE_DOC_KEY: row["image_doc_id"],
-            }
-
-            # Check if all standard conditions match
-            match = all(session.get(k) == v for k, v in standard_conditions.items())
-
-            # Apply date range filtering
-            if match and date_start:
-                session_start = session.get(self.START_KEY, "")
-                match = match and session_start >= date_start
-
-            if match and date_end:
-                session_start = session.get(self.START_KEY, "")
-                match = match and session_start <= date_end
-
-            if match:
-                results.append(session)
+            session_dict = dict(row)
+            # Parse the metadata JSON if it exists
+            if session_dict.get("metadata"):
+                session_dict["metadata"] = json.loads(session_dict["metadata"])
+            results.append(session_dict)
 
-        return results if results else None
+        return results
 
     def len_table(self, table_name: str) -> int:
         """Return the total number of rows in the specified table."""
@@ -314,20 +460,35 @@ class Database:
         result = cursor.fetchone()
         return result[0] if result and result[0] is not None else 0
 
-    def get_image(self, path: str) -> dict[str, Any] | None:
-        """Get an image record by path."""
+    def get_image(self, repo_url: str, path: str) -> ImageRow | None:
+        """Get an image record by repo_url and relative path.
+
+        Args:
+            repo_url: The repository URL
+            path: Path relative to the repository root
+
+        Returns:
+            Image record with relative path, repo_id, and repo_url, or None if not found
+        """
         cursor = self._db.cursor()
         cursor.execute(
-            f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE} WHERE path = ?",
-            (path,),
+            f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+            WHERE r.url = ? AND i.path = ?
+            """,
+            (repo_url, path),
         )
-        row = cursor.fetchone()
 
+        row = cursor.fetchone()
         if row is None:
             return None
 
         metadata = json.loads(row["metadata"])
         metadata["path"] = row["path"]
+        metadata["repo_id"] = row["repo_id"]
+        metadata["repo_url"] = row["repo_url"]
         metadata["id"] = row["id"]
 
         # Add date fields back to metadata for compatibility
@@ -338,17 +499,24 @@ class Database:
 
         return metadata
 
-    def all_images(self) -> list[dict[str, Any]]:
-        """Return all image records."""
+    def all_images(self) -> list[ImageRow]:
+        """Return all image records with relative paths, repo_id, and repo_url."""
         cursor = self._db.cursor()
         cursor.execute(
-            f"SELECT id, path, date_obs, date, metadata FROM {self.IMAGES_TABLE}"
+            f"""
+            SELECT i.id, i.repo_id, i.path, i.date_obs, i.date, i.metadata, r.url as repo_url
+            FROM {self.IMAGES_TABLE} i
+            JOIN {self.REPOS_TABLE} r ON i.repo_id = r.id
+            """
         )
 
         results = []
         for row in cursor.fetchall():
             metadata = json.loads(row["metadata"])
+            # Return relative path, repo_id, and repo_url for caller
             metadata["path"] = row["path"]
+            metadata["repo_id"] = row["repo_id"]
+            metadata["repo_url"] = row["repo_url"]
             metadata["id"] = row["id"]
 
             # Add date fields back to metadata for compatibility
@@ -361,35 +529,6 @@ class Database:
 
         return results
 
-    def all_sessions(self) -> list[dict[str, Any]]:
-        """Return all session records."""
-        cursor = self._db.cursor()
-        cursor.execute(
-            f"""
-            SELECT id, start, end, filter, imagetyp, object, telescop,
-                   num_images, exptime_total, image_doc_id
-            FROM {self.SESSIONS_TABLE}
-        """
-        )
-
-        results = []
-        for row in cursor.fetchall():
-            session = {
-                "id": row["id"],
-                self.START_KEY: row["start"],
-                self.END_KEY: row["end"],
-                self.FILTER_KEY: row["filter"],
-                self.IMAGETYP_KEY: row["imagetyp"],
-                self.OBJECT_KEY: row["object"],
-                self.TELESCOP_KEY: row["telescop"],
-                self.NUM_IMAGES_KEY: row["num_images"],
-                self.EXPTIME_TOTAL_KEY: row["exptime_total"],
-                self.IMAGE_DOC_KEY: row["image_doc_id"],
-            }
-            results.append(session)
-
-        return results
-
     def get_session_by_id(self, session_id: int) -> dict[str, Any] | None:
         """Get a session record by its ID.
 
@@ -414,20 +553,9 @@ class Database:
         if row is None:
             return None
 
-        return {
-            "id": row["id"],
-            self.START_KEY: row["start"],
-            self.END_KEY: row["end"],
-            self.FILTER_KEY: row["filter"],
-            self.IMAGETYP_KEY: row["imagetyp"],
-            self.OBJECT_KEY: row["object"],
-            self.TELESCOP_KEY: row["telescop"],
-            self.NUM_IMAGES_KEY: row["num_images"],
-            self.EXPTIME_TOTAL_KEY: row["exptime_total"],
-            self.IMAGE_DOC_KEY: row["image_doc_id"],
-        }
-
-    def get_session(self, to_find: dict[str, str]) -> dict[str, Any] | None:
+        return dict(row)
+
+    def get_session(self, to_find: dict[str, str]) -> SessionRow | None:
         """Find a session matching the given criteria.
 
         Searches for sessions with the same filter, image type, target, and telescope
@@ -470,21 +598,10 @@ class Database:
         if row is None:
             return None
 
-        return {
-            "id": row["id"],
-            self.START_KEY: row["start"],
-            self.END_KEY: row["end"],
-            self.FILTER_KEY: row["filter"],
-            self.IMAGETYP_KEY: row["imagetyp"],
-            self.OBJECT_KEY: row["object"],
-            self.TELESCOP_KEY: row["telescop"],
-            self.NUM_IMAGES_KEY: row["num_images"],
-            self.EXPTIME_TOTAL_KEY: row["exptime_total"],
-            self.IMAGE_DOC_KEY: row["image_doc_id"],
-        }
+        return dict(row)
 
     def upsert_session(
-        self, new: dict[str, Any], existing: dict[str, Any] | None = None
+        self, new: SessionRow, existing: SessionRow | None = None
     ) -> None:
         """Insert or update a session record."""
         cursor = self._db.cursor()

starbash/defaults/starbash.toml

@@ -3,6 +3,23 @@
 [repo]
 kind = "preferences"
 
+[aliases]
+# aliases can be used to map non standard (or non english) frame names to standard terms
+# This is also used to map filters based on common misspellings or variations.
+# We assume the first listed option in the list is the 'canonical' name used for printing etc...
+
+# frame types
+dark = ["dark", "darks"]
+flat = ["flat", "flats"]
+bias = ["bias", "biases"]
+
+# file suffixes
+fit = ["fits", "fit"]
+
+# filter names
+SiiOiii = ["SiiOiii", "SII-OIII", "S2-O3"]
+HaOiii = ["HaOiii", "HA-OIII", "Halpha-O3"]
+
 # FIXME, somewhere here list default patterns which can be used to identify NINA, ASIAIR, SEESTAR
 # raw repo layouts
 

starbash/main.py

@@ -6,7 +6,7 @@ import starbash.url as url
 import starbash
 
 from .app import Starbash, get_user_config_path, setup_logging
-from .commands import info, repo, select, user
+from .commands import info, process, repo, select, user
 from . import console
 
 app = typer.Typer(
@@ -17,6 +17,9 @@ app.add_typer(user.app, name="user", help="Manage user settings.")
 app.add_typer(repo.app, name="repo", help="Manage Starbash repositories.")
 app.add_typer(select.app, name="select", help="Manage session and target selection.")
 app.add_typer(info.app, name="info", help="Display system and data information.")
+app.add_typer(
+    process.app, name="process", help="Process images using automated workflows."
+)
 
 
 @app.callback(invoke_without_command=True)