thistle-db 0.6.0__py3-none-any.whl

thistle_db/__init__.py

@@ -0,0 +1,4 @@
+from thistle_db.api import get_tles
+from thistle_db.cli import main
+
+__all__ = ["get_tles", "main"]

thistle_db/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.6.0'
+__version_tuple__ = version_tuple = (0, 6, 0)
+
+__commit_id__ = commit_id = None

thistle_db/api.py

@@ -0,0 +1,61 @@
+"""Programmatic query API for the thistle-db database."""
+
+import datetime
+
+from sqlalchemy import select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from thistle_db.config import Settings, load_config
+from thistle_db.model import TLE, Base
+
+
+def open_session(config: Settings) -> tuple[Session, Engine]:
+    """Create the schema if needed and open a session on the configured database."""
+    engine = config.database.engine
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine)
+    return session_factory(), engine
+
+
+def tles_for_object(session: Session, satnum: int) -> list[TLE]:
+    """All TLEs for one satellite, ordered by epoch."""
+    stmt = select(TLE).where(TLE.norad_cat_id == satnum).order_by(TLE.epoch)
+    return list(session.execute(stmt).scalars().all())
+
+
+def nearest_tles_for_date(
+    session: Session, date: datetime.date, days: float
+) -> list[TLE]:
+    """Nearest TLE per satellite to 12:00 UTC on `date`, within +/- `days`."""
+    center = datetime.datetime.combine(date, datetime.time(12))
+    window = datetime.timedelta(days=days)
+    stmt = select(TLE).where(
+        TLE.epoch >= center - window,
+        TLE.epoch <= center + window,
+    )
+
+    nearest: dict[int, TLE] = {}
+    for tle in session.execute(stmt).scalars():
+        if tle.norad_cat_id is None:
+            continue
+        best = nearest.get(tle.norad_cat_id)
+        if best is None or abs(tle.epoch - center) < abs(best.epoch - center):
+            nearest[tle.norad_cat_id] = tle
+    return [nearest[satnum] for satnum in sorted(nearest)]
+
+
+def get_tles(satnum: int, config: Settings | None = None) -> list[tuple[str, str]]:
+    """Return (line1, line2) pairs for one satellite, ordered by epoch.
+
+    Opens a session on the configured database (config.toml / THISTLE_DB_*
+    env vars when `config` is None) and disposes it before returning.
+    """
+    if config is None:
+        config = load_config(None)
+    session, engine = open_session(config)
+    try:
+        return [(tle.line1, tle.line2) for tle in tles_for_object(session, satnum)]
+    finally:
+        session.close()
+        engine.dispose()

thistle_db/cli.py

@@ -0,0 +1,286 @@
+import datetime
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from loguru import logger
+from sgp4.alpha5 import from_alpha5
+
+from thistle_db.api import nearest_tles_for_date, open_session, tles_for_object
+from thistle_db.config import load_config
+from thistle_db.generator import generate as generate_outputs
+from thistle_db.ingest import FileStatus, ingest_source_file, ingest_sources
+
+app = typer.Typer(
+    name="thistle-db",
+    help="TLE database management tool",
+    no_args_is_help=True,
+)
+
+
+def _setup_logging(level: str) -> None:
+    logger.remove()
+    logger.add(sys.stderr, level=level.upper())
+
+
+CONFIG_TEMPLATE = """\
+# thistle-db configuration
+# See: https://github.com/jolsten/thistle
+
+# ----------------------------------------------
+# Database connection
+# ----------------------------------------------
+[database]
+# SQLAlchemy driver string. Examples:
+#   "sqlite"          - local SQLite file (default)
+#   "mysql+pymysql"   - MariaDB/MySQL via PyMySQL (install with: pip install thistle-db[mysql])
+drivername = "sqlite"
+
+# For SQLite: path to the database file (relative or absolute)
+# For MariaDB/MySQL: the database name
+name = "thistle-db.db"
+
+# Uncomment for MariaDB/MySQL:
+# host = "localhost"
+# port = 3306
+#
+# Credentials are loaded separately (never put passwords here).
+# Resolution order (highest priority first):
+#   1. Environment variables:  THISTLE_DB_DATABASE__USERNAME / THISTLE_DB_DATABASE__PASSWORD
+#   2. User secrets file:      ~/.config/thistle-db.toml
+#   3. System secrets file:    set secrets_file below
+#
+# secrets_file = "/etc/thistle-db/secrets.toml"
+
+# ----------------------------------------------
+# Ingest sources
+# ----------------------------------------------
+# Each [[ingest.sources]] entry defines a directory to scan for TLE/OMM files.
+# You can have multiple entries. File format is auto-detected by extension:
+#   .tle / .txt / .3le  ->  Two-Line Element format
+#   .json               ->  Space-Track OMM JSON
+#   .csv                ->  OMM CSV
+#   .xml                ->  OMM XML
+
+[[ingest.sources]]
+path = "./incoming"
+pattern = "*.tle"    # glob pattern for matching files
+
+# Add more sources as needed:
+# [[ingest.sources]]
+# path = "/data/spacetrack/daily"
+# pattern = "*.json"
+
+# ----------------------------------------------
+# Output generation
+# ----------------------------------------------
+[output]
+dir = "./output"     # directory where generated files are written
+
+# Which output formats to produce
+[output.formats]
+tle = true           # two-line element format (.tle files)
+omm = true           # OMM CSV format (.omm files)
+
+# Which output file types to generate
+[output.types]
+date_files = true    # one file per date: YYYYMMDD.{tle,omm}
+                     # contains the latest TLE per satellite for that date
+object_files = true  # one file per satellite: NORAD_ID.{tle,omm}
+                     # contains all TLEs for that satellite, ordered by epoch
+
+# ----------------------------------------------
+# Logging
+# ----------------------------------------------
+[logging]
+level = "INFO"       # DEBUG, INFO, WARNING, ERROR, CRITICAL
+"""
+
+SECRETS_TEMPLATE = """\
+# thistle-db user secrets
+# This file stores database credentials for your user account.
+# Keep this file private (chmod 600 on Linux/macOS).
+#
+# These values are overridden by environment variables:
+#   THISTLE_DB_DATABASE__USERNAME
+#   THISTLE_DB_DATABASE__PASSWORD
+
+username = ""
+password = ""
+"""
+
+
+@app.callback()
+def main_callback(
+    ctx: typer.Context,
+    config: Annotated[
+        Path,
+        typer.Option(
+            "-c",
+            "--config",
+            envvar="THISTLE_DB_CONFIG",
+            help="Path to config.toml (default: $THISTLE_DB_CONFIG or ./config.toml)",
+        ),
+    ] = Path("config.toml"),
+) -> None:
+    ctx.obj = config
+
+
+@app.command()
+def init(ctx: typer.Context) -> None:
+    """Scaffold config.toml and ~/.config/thistle-db.toml."""
+    config_path: Path = ctx.obj
+    secrets_path = Path.home() / ".config" / "thistle-db.toml"
+
+    created = []
+
+    if config_path.exists():
+        print(f"Config already exists: {config_path}")
+    else:
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        config_path.write_text(CONFIG_TEMPLATE)
+        created.append(str(config_path))
+
+    if secrets_path.exists():
+        print(f"Secrets file already exists: {secrets_path}")
+    else:
+        secrets_path.parent.mkdir(parents=True, exist_ok=True)
+        secrets_path.write_text(SECRETS_TEMPLATE)
+        created.append(str(secrets_path))
+
+    if created:
+        print("Created:")
+        for path in created:
+            print(f"  {path}")
+        print("\nEdit these files to configure your database and credentials.")
+    else:
+        print("Nothing to do - all files already exist.")
+
+
+@app.command()
+def ingest(
+    ctx: typer.Context,
+    files: Annotated[
+        Optional[list[Path]],
+        typer.Argument(
+            help="Specific files to ingest (if omitted, scans configured source dirs)"
+        ),
+    ] = None,
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force",
+            help="Re-ingest files even if their recorded state is unchanged",
+        ),
+    ] = False,
+) -> None:
+    """Ingest TLE/OMM files into the database."""
+    config = load_config(ctx.obj)
+    _setup_logging(config.logging.level)
+
+    session, engine = open_session(config)
+    try:
+        if files:
+            total = 0
+            failed = 0
+            for file in files:
+                # Explicitly named files always parse; state is still recorded.
+                status, count = ingest_source_file(session, file, force=True)
+                if status == FileStatus.FAILED:
+                    failed += 1
+                total += count
+            logger.info(
+                f"Ingested {total} new records from {len(files)} files"
+                + (f" ({failed} failed)" if failed else "")
+            )
+        else:
+            total = ingest_sources(session, config.ingest.sources, force=force)
+            logger.info(f"Ingested {total} new records from configured sources")
+    finally:
+        session.close()
+        engine.dispose()
+
+
+@app.command()
+def generate(ctx: typer.Context) -> None:
+    """Generate output TLE/OMM files from the database."""
+    config = load_config(ctx.obj)
+    _setup_logging(config.logging.level)
+
+    session, engine = open_session(config)
+    try:
+        generate_outputs(session, config.output)
+    finally:
+        session.close()
+        engine.dispose()
+
+
+def _parse_target(value: str) -> int | datetime.date:
+    """Parse a get-tle target: 8-digit YYYYMMDD -> date, else alpha-5 NORAD ID."""
+    if len(value) == 8 and value.isdigit():
+        try:
+            return datetime.datetime.strptime(value, "%Y%m%d").date()
+        except ValueError:
+            raise typer.BadParameter(
+                f"{value!r} is not a valid YYYYMMDD date"
+            ) from None
+    try:
+        return from_alpha5(value.upper())
+    except ValueError:
+        raise typer.BadParameter(
+            f"{value!r} is neither an alpha-5 NORAD ID nor a YYYYMMDD date"
+        ) from None
+
+
+@app.command(name="get-tle")
+def get_tle(
+    ctx: typer.Context,
+    target: Annotated[
+        str,
+        typer.Argument(
+            help="NORAD ID (alpha-5, e.g. 25544 or E5693) or date (YYYYMMDD)",
+            show_default=False,
+        ),
+    ],
+    days: Annotated[
+        float,
+        typer.Option(
+            "--days",
+            "-d",
+            help="Date mode: search window around the date, in days",
+        ),
+    ] = 7.0,
+) -> None:
+    """Print TLEs to stdout.
+
+    With a NORAD ID (alpha-5): all TLEs for that object, ordered by epoch.
+    With a YYYYMMDD date: the nearest TLE per object to 12:00 UTC on that
+    date, within +/- DAYS days.
+    """
+    parsed = _parse_target(target)
+
+    config = load_config(ctx.obj)
+    _setup_logging(config.logging.level)
+
+    session, engine = open_session(config)
+    try:
+        if isinstance(parsed, int):
+            tles = tles_for_object(session, parsed)
+        else:
+            tles = nearest_tles_for_date(session, parsed, days)
+
+        for tle in tles:
+            print(tle.line1)
+            print(tle.line2)
+    finally:
+        session.close()
+        engine.dispose()
+
+    if not tles:
+        logger.warning(f"No TLEs found for {target}")
+        raise typer.Exit(code=1)
+
+
+def main() -> None:
+    app()

thistle_db/config.py

@@ -0,0 +1,118 @@
+import tomllib
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from sqlalchemy import Engine, create_engine
+from sqlalchemy.engine import URL
+
+
+class Database(BaseModel):
+    drivername: str = "sqlite"
+    username: Optional[str] = None
+    password: Optional[str] = None
+    host: Optional[str] = None
+    port: Optional[int] = None
+    name: Optional[str] = ":memory:"
+    secrets_file: Optional[str] = None
+
+    @property
+    def url(self) -> URL:
+        return URL.create(
+            drivername=self.drivername,
+            username=self.username,
+            password=self.password,
+            host=self.host,
+            port=self.port,
+            database=self.name,
+        )
+
+    @property
+    def engine(self) -> Engine:
+        return create_engine(self.url)
+
+
+class IngestSource(BaseModel):
+    path: str
+    pattern: str = "*.tle"
+
+
+class IngestConfig(BaseModel):
+    sources: list[IngestSource] = []
+
+
+class OutputFormats(BaseModel):
+    tle: bool = True
+    omm: bool = True
+
+
+class OutputTypes(BaseModel):
+    date_files: bool = True
+    object_files: bool = True
+
+
+class OutputConfig(BaseModel):
+    dir: str = "./output"
+    formats: OutputFormats = OutputFormats()
+    types: OutputTypes = OutputTypes()
+
+
+class LoggingConfig(BaseModel):
+    level: str = "INFO"
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="THISTLE_DB_",
+        env_nested_delimiter="__",
+    )
+
+    database: Database = Database()
+    ingest: IngestConfig = IngestConfig()
+    output: OutputConfig = OutputConfig()
+    logging: LoggingConfig = LoggingConfig()
+
+
+def load_config(path: Path | None = None) -> Settings:
+    """Load settings from TOML file with layered credential resolution.
+
+    Resolution order (highest priority first):
+    1. Environment variables (THISTLE_DB_DATABASE__USERNAME, etc.)
+    2. User secrets (~/.config/thistle-db.toml)
+    3. System secrets file (database.secrets_file in config)
+    4. Values in config.toml
+    """
+    toml_data: dict = {}
+
+    if path is not None and path.exists():
+        with open(path, "rb") as f:
+            toml_data = tomllib.load(f)
+
+    # Load system secrets file if specified
+    db_data = toml_data.get("database", {})
+    secrets_file = db_data.get("secrets_file")
+    if secrets_file:
+        secrets_path = Path(secrets_file)
+        if secrets_path.exists():
+            with open(secrets_path, "rb") as f:
+                secrets = tomllib.load(f)
+            # Merge secrets into database config (secrets_file has lower priority
+            # than values already in db_data, which have lower priority than env vars)
+            for key in ("username", "password"):
+                if key in secrets and key not in db_data:
+                    db_data[key] = secrets[key]
+            toml_data["database"] = db_data
+
+    # Load user-local secrets (~/.config/thistle-db.toml)
+    user_secrets_path = Path.home() / ".config" / "thistle-db.toml"
+    if user_secrets_path.exists():
+        with open(user_secrets_path, "rb") as f:
+            user_secrets = tomllib.load(f)
+        for key in ("username", "password"):
+            if key in user_secrets and key not in db_data:
+                db_data[key] = user_secrets[key]
+        toml_data["database"] = db_data
+
+    # pydantic-settings handles env var overrides automatically
+    return Settings(**toml_data)

thistle_db/generator.py

@@ -0,0 +1,155 @@
+import datetime
+import pathlib
+
+from loguru import logger
+from sgp4.api import Satrec
+from sgp4.exporter import export_omm
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from thistle_db.config import OutputConfig
+from thistle_db.model import TLE
+from thistle_db.reader import TLETuple, write_omm_csv, write_tle
+
+
+def _reconstruct_satrec(tle: TLE) -> Satrec:
+    """Reconstruct a Satrec object from stored TLE lines."""
+    return Satrec.twoline2rv(tle.line1, tle.line2)
+
+
+def _safe_export_omm(sat: Satrec, name: str) -> dict | None:
+    """Export OMM dict from Satrec, returning None if export fails."""
+    try:
+        return export_omm(sat, name)
+    except (ValueError, AttributeError):
+        return None
+
+
+def _get_object_name(tle: TLE) -> str:
+    """Get object name from OmmMetadata if available, else fall back to object_id."""
+    if tle.omm_metadata and tle.omm_metadata.object_name:
+        return tle.omm_metadata.object_name
+    return tle.object_id
+
+
+def generate_date_files(
+    session: Session, output_dir: pathlib.Path, config: OutputConfig
+) -> None:
+    """Generate one file per date with the latest TLE per object."""
+    # Get distinct dates (UTC date truncation of epoch)
+    date_query = select(func.date(TLE.epoch).label("d")).distinct()
+    dates = session.execute(date_query).scalars().all()
+
+    logger.info(f"Generating date files for {len(dates)} dates")
+
+    for raw_date in dates:
+        if raw_date is None:
+            continue
+
+        # func.date() returns str on SQLite and datetime.date on MariaDB/MySQL.
+        if isinstance(raw_date, datetime.date):
+            date_val = raw_date
+        else:
+            date_val = datetime.date.fromisoformat(str(raw_date))
+
+        # Subquery: max epoch per norad_cat_id on this date
+        subq = (
+            select(
+                TLE.norad_cat_id,
+                func.max(TLE.epoch).label("max_epoch"),
+            )
+            .where(func.date(TLE.epoch) == date_val)
+            .group_by(TLE.norad_cat_id)
+            .subquery()
+        )
+
+        # Join back to get full TLE rows
+        stmt = (
+            select(TLE)
+            .join(
+                subq,
+                (TLE.norad_cat_id == subq.c.norad_cat_id)
+                & (TLE.epoch == subq.c.max_epoch),
+            )
+            .order_by(TLE.norad_cat_id)
+        )
+        tles = session.execute(stmt).scalars().all()
+
+        if not tles:
+            continue
+
+        filename_date = date_val.strftime("%Y%m%d")
+
+        # TLE output
+        if config.formats.tle:
+            tle_path = output_dir / f"{filename_date}.tle"
+            tle_tuples: list[TLETuple] = [(t.line1, t.line2) for t in tles]
+            write_tle(tle_path, tle_tuples)
+
+        # OMM output
+        if config.formats.omm:
+            omm_path = output_dir / f"{filename_date}.omm"
+            omm_records = []
+            for tle in tles:
+                sat = _reconstruct_satrec(tle)
+                name = _get_object_name(tle)
+                record = _safe_export_omm(sat, name)
+                if record is not None:
+                    omm_records.append(record)
+            write_omm_csv(omm_path, omm_records)
+
+
+def generate_object_files(
+    session: Session, output_dir: pathlib.Path, config: OutputConfig
+) -> None:
+    """Generate one file per satellite with all its TLEs."""
+    # Get distinct NORAD IDs
+    norad_ids = (
+        session.execute(select(TLE.norad_cat_id).distinct().order_by(TLE.norad_cat_id))
+        .scalars()
+        .all()
+    )
+
+    logger.info(f"Generating object files for {len(norad_ids)} satellites")
+
+    for norad_id in norad_ids:
+        if norad_id is None:
+            continue
+
+        stmt = select(TLE).where(TLE.norad_cat_id == norad_id).order_by(TLE.epoch)
+        tles = session.execute(stmt).scalars().all()
+
+        if not tles:
+            continue
+
+        # TLE output
+        if config.formats.tle:
+            tle_path = output_dir / f"{norad_id}.tle"
+            tle_tuples: list[TLETuple] = [(t.line1, t.line2) for t in tles]
+            write_tle(tle_path, tle_tuples)
+
+        # OMM output
+        if config.formats.omm:
+            omm_path = output_dir / f"{norad_id}.omm"
+            omm_records = []
+            for tle in tles:
+                sat = _reconstruct_satrec(tle)
+                name = _get_object_name(tle)
+                record = _safe_export_omm(sat, name)
+                if record is not None:
+                    omm_records.append(record)
+            write_omm_csv(omm_path, omm_records)
+
+
+def generate(session: Session, config: OutputConfig) -> None:
+    """Generate all configured output files."""
+    output_dir = pathlib.Path(config.dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if config.types.date_files:
+        generate_date_files(session, output_dir, config)
+
+    if config.types.object_files:
+        generate_object_files(session, output_dir, config)
+
+    logger.info(f"Output generation complete → {output_dir}")