fgsmdb 0.1.0__py3-none-any.whl

fgsmdb/__init__.py

@@ -0,0 +1,3 @@
+"""fgsmdb — FlightGear Scenemodels Database Tools."""
+
+__version__ = "0.1.0"

fgsmdb/__main__.py

@@ -0,0 +1,34 @@
+"""CLI entry point for fgsmdb.
+
+Usage:
+    python3 -m fgsmdb export [options]
+"""
+
+import argparse
+import sys
+
+from .commands import export
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="fgsmdb",
+        description="FlightGear Scenemodels Database Tools",
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Register the export subcommand
+    export.register(subparsers)
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    # Dispatch to the command handler
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

fgsmdb/commands/__init__.py

@@ -0,0 +1 @@
+# fgsmdb command modules

fgsmdb/commands/export.py

@@ -0,0 +1,180 @@
+"""Export command — orchestrates the full scenemodels export pipeline.
+
+Usage:
+    python3 -m fgsmdb export [--env-file .env] [--steps sanitize,gndelev,export,mkdiridx,upload,distribute]
+    python3 -m fgsmdb export --all
+    python3 -m fgsmdb export --steps export,mkdiridx
+
+Available steps:
+    sanitize   - Fix data integrity (tile numbers, elevation offsets)
+    gndelev    - Compute ground elevations using fgelev
+    export     - Export models and STG files from the database
+    mkdiridx   - Create .dirindex files and Airports archive
+    upload     - Sync export directory to local scenery tree
+    distribute - Rsync scenery to mirror servers
+"""
+
+import logging
+import os
+import tarfile
+
+from ..config import load_config
+from ..db import Database
+from ..steps import sanitize, gndelev, export_models, export_stg, dirindex, sync, rsync_distribute
+
+ALL_STEPS = ["sanitize", "gndelev", "export", "mkdiridx", "upload"]
+VALID_STEPS = ALL_STEPS + ["distribute"]
+
+logger = logging.getLogger(__name__)
+
+
+def register(subparsers):
+    """Register the export subcommand with the argument parser."""
+    parser = subparsers.add_parser(
+        "export",
+        help="Export scenemodels database to TerraSync file tree",
+        description="Export the FlightGear scenemodels database into a "
+                    "TerraSync-compatible file tree and optionally distribute to mirrors.",
+    )
+    parser.add_argument(
+        "--env-file",
+        default=None,
+        help="Path to .env file (default: auto-detect)",
+    )
+    parser.add_argument(
+        "--steps",
+        default=None,
+        help="Comma-separated list of steps to run (default: all except distribute)",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Run all steps (equivalent to: %s)" % ",".join(ALL_STEPS),
+    )
+    parser.add_argument(
+        "--skip-check",
+        action="store_true",
+        help="Skip the pending-updates threshold check",
+    )
+    parser.add_argument(
+        "--min-updates",
+        type=int,
+        default=None,
+        help="Override MIN_UPDATES threshold from .env",
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    parser.set_defaults(func=run)
+
+
+def run(args):
+    """Execute the export pipeline."""
+    # Setup logging
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    # Load configuration
+    config = load_config(args.env_file)
+
+    # Determine which steps to run
+    if args.steps:
+        steps = [s.strip() for s in args.steps.split(",")]
+        for s in steps:
+            if s not in VALID_STEPS:
+                logger.error("Unknown step: %s. Valid steps: %s", s, ", ".join(VALID_STEPS))
+                raise SystemExit(1)
+    else:
+        steps = ALL_STEPS
+
+    # Override min_updates if specified
+    min_updates = args.min_updates if args.min_updates is not None else config["min_updates"]
+
+    # Connect to the database
+    db = Database(config["pguri"])
+
+    try:
+        # Always update statistics first
+        logger.info("Updating statistics...")
+        db.execute("SELECT fn_update_statistics()")
+
+        # Check if there are enough pending updates
+        if not args.skip_check:
+            rows = db.execute(
+                "SELECT count(*) AS cnt FROM fgs_objects WHERE ob_gndelev = -9999"
+            )
+            count = rows[0]["cnt"] if rows else 0
+            logger.info("Pending model updates: %d (minimum: %d)", count, min_updates)
+            if count < min_updates:
+                logger.info("Not enough pending updates. Exiting.")
+                return
+
+        # Run pipeline steps
+        if "sanitize" in steps:
+            sanitize.run(db)
+
+        if "gndelev" in steps:
+            gndelev.run(
+                db,
+                fgelev_path=config["fgelev_path"],
+                fg_root=config["fg_root"],
+                fg_scenery=config["fg_scenery"],
+            )
+
+        if "export" in steps:
+            export_models.run(db, target=config["fg_scenery_export"])
+            export_stg.run(db, target=config["fg_scenery_export"])
+
+        if "mkdiridx" in steps:
+            dirindex.create_root_dirindex(
+                fg_scenery=config["fg_scenery"],
+                fg_scenery_export=config["fg_scenery_export"],
+            )
+
+        if "upload" in steps:
+            sync.run(
+                fg_scenery_export=config["fg_scenery_export"],
+                fg_scenery=config["fg_scenery"],
+            )
+
+        # Update statistics after export
+        if any(s in steps for s in ("export", "upload")):
+            db.execute("SELECT fn_update_statistics()")
+
+        # Create SharedModels tarball
+        if "upload" in steps:
+            _create_shared_models_tarball(config["fg_scenery"])
+
+        # Distribution (rsync to mirrors)
+        if "distribute" in steps:
+            rsync_distribute.run(
+                fg_scenery=config["fg_scenery"],
+                sync_targets=config["sync_targets"],
+                cdn_invalidate_cmd=config["cdn_invalidate_cmd"],
+            )
+
+        logger.info("Export pipeline complete.")
+
+    finally:
+        db.close()
+
+
+def _create_shared_models_tarball(fg_scenery: str):
+    """Create SharedModels.txz from the Models directory."""
+    models_path = os.path.join(fg_scenery, "Models")
+    output_path = os.path.join(fg_scenery, "SharedModels.txz")
+
+    if not os.path.isdir(models_path):
+        logger.warning("No Models directory at %s, skipping tarball.", models_path)
+        return
+
+    logger.info("Creating SharedModels.txz...")
+    with tarfile.open(output_path, "w:xz") as tar:
+        tar.add(models_path, arcname="Models")
+    logger.info("SharedModels.txz created at %s", output_path)

fgsmdb/config.py

@@ -0,0 +1,37 @@
+"""Configuration loader — reads .env and provides settings to all modules."""
+
+import os
+from dotenv import load_dotenv
+
+
+def load_config(env_file: str | None = None) -> dict:
+    """Load configuration from .env file and environment variables.
+
+    Args:
+        env_file: Path to .env file. If None, searches in current directory
+                  and parent directories.
+
+    Returns:
+        Dictionary with all configuration values.
+    """
+    if env_file:
+        load_dotenv(env_file)
+    else:
+        load_dotenv()
+
+    config = {
+        "pguri": os.environ["PGURI"],
+        "fgelev_path": os.environ.get("FGELEV_PATH", "/usr/local/bin/fgelev"),
+        "fg_root": os.environ.get("FG_ROOT", "/app/data"),
+        "fg_scenery": os.environ.get("FG_SCENERY", "./fgscenery"),
+        "fg_scenery_export": os.environ.get("FG_SCENERY_EXPORT", "./export"),
+        "groundnets_path": os.environ.get("GROUNDNETS_PATH", "./groundnets"),
+        "min_updates": int(os.environ.get("MIN_UPDATES", "1")),
+        "sync_targets": [
+            t.strip()
+            for t in os.environ.get("SYNC_TARGETS", "").split(",")
+            if t.strip()
+        ],
+        "cdn_invalidate_cmd": os.environ.get("CDN_INVALIDATE_CMD", ""),
+    }
+    return config

fgsmdb/db.py

@@ -0,0 +1,66 @@
+"""Database connection helper with automatic reconnect."""
+
+import logging
+import psycopg2
+import psycopg2.extras
+
+logger = logging.getLogger(__name__)
+
+
+class Database:
+    """Thin wrapper around psycopg2 with reconnect logic."""
+
+    def __init__(self, pguri: str):
+        self.pguri = pguri
+        self._conn = None
+
+    def connect(self):
+        """Establish database connection."""
+        if self._conn is None or self._conn.closed:
+            logger.info("Connecting to database...")
+            self._conn = psycopg2.connect(self.pguri)
+            self._conn.autocommit = True
+        return self._conn
+
+    @property
+    def conn(self):
+        return self.connect()
+
+    def cursor(self):
+        """Return a DictCursor, reconnecting if needed."""
+        return self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+
+    def execute(self, sql: str, params=None, retries: int = 3):
+        """Execute a query with automatic reconnect on OperationalError.
+
+        Returns:
+            List of result rows, or empty list for non-SELECT statements.
+        """
+        for attempt in range(retries):
+            try:
+                cur = self.cursor()
+                cur.execute(sql, params)
+                if cur.description:
+                    return cur.fetchall()
+                return []
+            except psycopg2.OperationalError:
+                logger.warning(
+                    "Database connection lost (attempt %d/%d), reconnecting...",
+                    attempt + 1,
+                    retries,
+                )
+                self._conn = None
+                if attempt == retries - 1:
+                    raise
+
+    def execute_batch(self, statements: list[str]):
+        """Execute multiple SQL statements in sequence."""
+        cur = self.cursor()
+        for stmt in statements:
+            cur.execute(stmt)
+
+    def close(self):
+        """Close the database connection."""
+        if self._conn and not self._conn.closed:
+            self._conn.close()
+            self._conn = None

fgsmdb/steps/__init__.py

@@ -0,0 +1 @@
+# fgsmdb pipeline step modules

fgsmdb/steps/dirindex.py

@@ -0,0 +1,147 @@
+"""Directory index generation — creates .dirindex files for TerraSync.
+
+Equivalent to the legacy CreateDirectoryIndexes.py and mkdiridx.sh scripts.
+
+TerraSync .dirindex format:
+    version:1
+    path:<relative_path>
+    f:<filename>:<sha1>:<size>     (regular files)
+    t:<filename>:<sha1>:<size>     (tar archives)
+    d:<dirname>:<sha1_of_dirindex> (subdirectories)
+"""
+
+import hashlib
+import logging
+import os
+import tarfile
+from datetime import datetime, timezone
+
+logger = logging.getLogger(__name__)
+
+DIRINDEX_VERSION = 1
+DIRINDEX_FILENAME = ".dirindex"
+
+
+def sha1_of_file(path: str) -> str:
+    """Compute SHA1 hash of a file."""
+    h = hashlib.sha1()
+    try:
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                h.update(chunk)
+    except OSError:
+        pass
+    return h.hexdigest()
+
+
+def create_dirindex(path: str, parent: str = "") -> str:
+    """Recursively create .dirindex files in a directory tree.
+
+    Args:
+        path: Absolute or relative path to the directory.
+        parent: Relative path prefix for the dirindex 'path:' field.
+
+    Returns:
+        SHA1 hash of the generated .dirindex file.
+    """
+    dirindex_path = os.path.join(path, DIRINDEX_FILENAME)
+
+    entries = sorted(os.listdir(path))
+
+    with open(dirindex_path, "w") as f:
+        f.write(f"version:{DIRINDEX_VERSION}\n")
+        f.write(f"path:{parent}\n")
+
+        for entry in entries:
+            full_path = os.path.join(path, entry)
+
+            if entry == DIRINDEX_FILENAME:
+                continue
+
+            if os.path.isfile(full_path):
+                file_hash = sha1_of_file(full_path)
+                file_size = os.stat(full_path).st_size
+                if entry.endswith(".tgz") or entry.endswith(".txz"):
+                    f.write(f"t:{entry}:{file_hash}:{file_size}\n")
+                else:
+                    f.write(f"f:{entry}:{file_hash}:{file_size}\n")
+
+            elif os.path.isdir(full_path) and entry != ".svn":
+                child_parent = os.path.join(parent, entry) if parent else entry
+                child_hash = create_dirindex(full_path, child_parent)
+                f.write(f"d:{entry}:{child_hash}\n")
+
+    return sha1_of_file(dirindex_path)
+
+
+def create_root_dirindex(fg_scenery: str, fg_scenery_export: str):
+    """Create the root .dirindex and Airports archive.
+
+    This combines data from the remote scenery (Airports, Terrain) with
+    exported data (Objects, Models) into a unified root .dirindex.
+
+    Args:
+        fg_scenery: Path to the remote/mounted scenery (has Airports, Terrain).
+        fg_scenery_export: Path to the export directory (has Objects, Models).
+    """
+    logger.info("Creating directory indexes...")
+
+    # Create Airports dirindex and archive
+    airports_path = os.path.join(fg_scenery, "Airports")
+    if os.path.isdir(airports_path):
+        logger.info("Creating Airports .dirindex...")
+        create_dirindex(airports_path, "Airports")
+
+        # Create Airports archive
+        archive_path = os.path.join(fg_scenery, "Airports_archive.tgz")
+        logger.info("Creating Airports_archive.tgz...")
+        with tarfile.open(archive_path, "w:gz") as tar:
+            tar.add(airports_path, arcname="Airports")
+
+    # Create Objects and Models dirindexes in the export directory
+    objects_path = os.path.join(fg_scenery_export, "Objects")
+    if os.path.isdir(objects_path):
+        logger.info("Creating Objects .dirindex...")
+        create_dirindex(objects_path, "Objects")
+
+    models_path = os.path.join(fg_scenery_export, "Models")
+    if os.path.isdir(models_path):
+        logger.info("Creating Models .dirindex...")
+        create_dirindex(models_path, "Models")
+
+    # Build the root .dirindex
+    logger.info("Creating root .dirindex...")
+    root_dirindex_path = os.path.join(fg_scenery_export, DIRINDEX_FILENAME)
+    with open(root_dirindex_path, "w") as f:
+        f.write(f"version:{DIRINDEX_VERSION}\n")
+        f.write("path:\n")
+        f.write(f"time:{datetime.now(timezone.utc).strftime('%Y%m%d-%H:%MZ')}\n")
+
+        # Airports (from remote scenery)
+        airports_dirindex = os.path.join(airports_path, DIRINDEX_FILENAME)
+        if os.path.isfile(airports_dirindex):
+            f.write(f"d:Airports:{sha1_of_file(airports_dirindex)}\n")
+
+        # Models (from export)
+        models_dirindex = os.path.join(models_path, DIRINDEX_FILENAME)
+        if os.path.isfile(models_dirindex):
+            f.write(f"d:Models:{sha1_of_file(models_dirindex)}\n")
+
+        # Objects (from export)
+        objects_dirindex = os.path.join(objects_path, DIRINDEX_FILENAME)
+        if os.path.isfile(objects_dirindex):
+            f.write(f"d:Objects:{sha1_of_file(objects_dirindex)}\n")
+
+        # Terrain (from remote scenery)
+        terrain_dirindex = os.path.join(fg_scenery, "Terrain", DIRINDEX_FILENAME)
+        if os.path.isfile(terrain_dirindex):
+            f.write(f"d:Terrain:{sha1_of_file(terrain_dirindex)}\n")
+
+        # Airports archive tarball
+        archive_path = os.path.join(fg_scenery, "Airports_archive.tgz")
+        if os.path.isfile(archive_path):
+            archive_hash = sha1_of_file(archive_path)
+            archive_size = os.stat(archive_path).st_size
+            f.write(f"t:Airports_archive.tgz:{archive_hash}:{archive_size}\n")
+
+    logger.info("Root .dirindex created at %s", root_dirindex_path)

fgsmdb/steps/export_models.py

@@ -0,0 +1,104 @@
+"""Export models step — extract 3D model files from the database.
+
+Equivalent to the legacy exportmodels.py script.
+Exports shared models (to Models/) and per-hemisphere object models (to Objects/).
+"""
+
+import base64
+import io
+import logging
+import os
+import tarfile
+from ..db import Database
+
+logger = logging.getLogger(__name__)
+
+
+def run(db: Database, target: str, incremental: bool = False):
+    """Export all models: shared models + east/west hemisphere objects.
+
+    Args:
+        db: Database connection wrapper.
+        target: Target directory for exported files.
+        incremental: If True, only export when new models exist.
+    """
+    os.makedirs(target, exist_ok=True)
+
+    if incremental and not _has_new_models(db):
+        logger.info("No new models to export (incremental mode).")
+        return
+
+    _export_shared_models(db, target)
+    _export_hemisphere_models(db, target, "west")
+    _export_hemisphere_models(db, target, "east")
+
+
+def _has_new_models(db: Database) -> bool:
+    """Check if there are new models since the last export timestamp."""
+    rows = db.execute(
+        """SELECT count(*) AS cnt FROM fgs_models
+           WHERE mo_modified > (SELECT max(ti_stamp) FROM fgs_timestamps WHERE ti_type=1)"""
+    )
+    return rows[0]["cnt"] > 0 if rows else False
+
+
+def _export_shared_models(db: Database, target: str):
+    """Export shared models (mo_shared > 0) to Models/ subdirectory."""
+    logger.info("Exporting shared models...")
+
+    rows = db.execute(
+        """SELECT m.mo_id AS id,
+                  concat('Models/', g.mg_path) AS path,
+                  m.mo_modelfile AS modelfile
+           FROM fgs_models AS m
+           LEFT JOIN fgs_modelgroups AS g ON m.mo_shared = g.mg_id
+           WHERE m.mo_shared > 0"""
+    )
+
+    count = 0
+    for row in rows:
+        _extract_model(row, target)
+        count += 1
+
+    logger.info("Exported %d shared models.", count)
+
+
+def _export_hemisphere_models(db: Database, target: str, hemisphere: str):
+    """Export per-object models for a hemisphere.
+
+    Args:
+        hemisphere: "east" or "west"
+    """
+    constraint = ">= 0" if hemisphere == "east" else "< 0"
+    logger.info("Exporting %s hemisphere models...", hemisphere)
+
+    rows = db.execute(
+        f"""SELECT m.mo_id AS id,
+                   concat('Objects/', fn_SceneDir(o.wkb_geometry), '/', fn_SceneSubDir(o.wkb_geometry), '/') AS path,
+                   m.mo_modelfile AS modelfile
+            FROM fgs_objects AS o
+            LEFT JOIN fgs_models AS m ON o.ob_model = m.mo_id
+            WHERE o.ob_gndelev > -9999 AND m.mo_shared = 0 AND ST_X(o.wkb_geometry) {constraint}"""
+    )
+
+    count = 0
+    for row in rows:
+        _extract_model(row, target)
+        count += 1
+
+    logger.info("Exported %d %s hemisphere models.", count, hemisphere)
+
+
+def _extract_model(row, target: str):
+    """Decode and extract a model's tar.gz archive to the target path."""
+    model_path = row["path"]
+    full_path = os.path.join(target, model_path)
+
+    os.makedirs(full_path, exist_ok=True)
+
+    try:
+        tar_bytes = io.BytesIO(base64.b64decode(row["modelfile"]))
+        with tarfile.open(fileobj=tar_bytes, mode="r:gz") as tar:
+            tar.extractall(path=full_path)
+    except Exception as e:
+        logger.warning("Failed to extract model %s: %s", row["id"], e)

fgsmdb/steps/export_stg.py

@@ -0,0 +1,57 @@
+"""Export STG step — generate .stg files (object placement) for each scenery tile.
+
+Equivalent to the legacy exportstg.py script.
+"""
+
+import logging
+import os
+from ..db import Database
+
+logger = logging.getLogger(__name__)
+
+
+def run(db: Database, target: str):
+    """Generate all .stg files from the database.
+
+    Queries all distinct tiles that have objects or signs, then generates
+    an .stg file for each tile using the database function fn_DumpStgRows().
+
+    Args:
+        db: Database connection wrapper.
+        target: Target directory for exported files.
+    """
+    logger.info("Querying tile list...")
+
+    sql_path = "concat('Objects/', fn_SceneDir(wkb_geometry), '/', fn_SceneSubDir(wkb_geometry), '/') AS obpath"
+    rows = db.execute(
+        f"""SELECT DISTINCT fn_gettilenumber(o.wkb_geometry) AS tile, {sql_path}
+            FROM fgs_objects AS o
+            UNION
+            SELECT DISTINCT fn_gettilenumber(s.wkb_geometry) AS tile, {sql_path}
+            FROM fgs_signs AS s
+            ORDER BY tile"""
+    )
+
+    num_tiles = len(rows)
+    logger.info("Generating .stg files for %d tiles...", num_tiles)
+
+    for i, row in enumerate(rows, 1):
+        ob_path = row["obpath"]
+        ob_tile = row["tile"]
+        stg_file = f"{ob_tile}.stg"
+
+        if i % 1000 == 0:
+            logger.info("Progress: %d / %d tiles", i, num_tiles)
+
+        # Get STG content from the database function
+        stg_rows = db.execute("SELECT fn_DumpStgRows(%s)", (ob_tile,))
+
+        if stg_rows and stg_rows[0][0]:
+            parent = os.path.join(target, ob_path)
+            os.makedirs(parent, exist_ok=True)
+
+            stg_full_path = os.path.join(parent, stg_file)
+            with open(stg_full_path, "w") as f:
+                f.write(f"{stg_rows[0][0]}\n")
+
+    logger.info("STG export complete: %d tiles processed.", num_tiles)

fgsmdb/steps/gndelev.py

@@ -0,0 +1,107 @@
+"""Ground elevation step — compute ground elevation for new objects using fgelev.
+
+Equivalent to the legacy gndelev.sh script.
+
+This step requires:
+- fgelev binary (from FlightGear/SimGear)
+- Terrain data in FG_SCENERY
+- FG_ROOT pointing to FlightGear data (defaults.xml)
+"""
+
+import logging
+import os
+import subprocess
+from ..db import Database
+
+logger = logging.getLogger(__name__)
+
+
+def run(db: Database, fgelev_path: str, fg_root: str, fg_scenery: str):
+    """Compute ground elevations for objects with ob_gndelev = -9999.
+
+    Queries unprocessed objects from the DB, pipes their coordinates through
+    fgelev, then updates the database with computed elevations.
+
+    Args:
+        db: Database connection wrapper.
+        fgelev_path: Path to the fgelev binary.
+        fg_root: FG_ROOT path (FlightGear data directory).
+        fg_scenery: FG_SCENERY path (terrain data directory).
+    """
+    terrain_path = os.path.join(fg_scenery, "Terrain")
+    if not os.path.isdir(terrain_path):
+        logger.error("No Terrain found in %s. Skipping gndelev.", fg_scenery)
+        return
+
+    logger.info("Querying objects needing ground elevation...")
+    rows = db.execute(
+        """SELECT ob_id, ST_X(wkb_geometry) AS lon, ST_Y(wkb_geometry) AS lat
+           FROM fgs_objects
+           WHERE ob_valid IS true AND ob_gndelev = -9999
+           ORDER BY ob_tile"""
+    )
+
+    if not rows:
+        logger.info("No objects need ground elevation updates.")
+        return
+
+    logger.info("Computing ground elevation for %d objects...", len(rows))
+
+    # Prepare input for fgelev: "id lon lat\n" per line
+    fgelev_input = ""
+    for row in rows:
+        fgelev_input += f"{row['ob_id']} {row['lon']} {row['lat']}\n"
+
+    # Run fgelev
+    env = os.environ.copy()
+    env["FG_ROOT"] = fg_root
+    env["FG_SCENERY"] = fg_scenery
+
+    try:
+        result = subprocess.run(
+            [fgelev_path],
+            input=fgelev_input,
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+    except FileNotFoundError:
+        logger.error("fgelev binary not found at %s", fgelev_path)
+        return
+
+    # fgelev may return non-zero on EOF, that's expected
+    if result.stdout:
+        _process_fgelev_output(db, result.stdout)
+    else:
+        logger.warning("fgelev produced no output. stderr: %s", result.stderr[:500])
+
+
+def _process_fgelev_output(db: Database, output: str):
+    """Parse fgelev output and update the database.
+
+    fgelev output format: "id:elevation\n"
+    """
+    update_count = 0
+    for line in output.strip().split("\n"):
+        line = line.strip()
+        if not line or ":" not in line:
+            continue
+
+        parts = line.split(":")
+        if len(parts) != 2:
+            continue
+
+        try:
+            ob_id = int(parts[0].strip())
+            elevation = float(parts[1].strip())
+        except ValueError:
+            logger.warning("Could not parse fgelev output line: %s", line)
+            continue
+
+        db.execute(
+            "UPDATE fgs_objects SET ob_gndelev = %s WHERE ob_id = %s AND ob_gndelev = -9999",
+            (elevation, ob_id),
+        )
+        update_count += 1
+
+    logger.info("Updated ground elevation for %d objects.", update_count)

fgsmdb/steps/rsync_distribute.py

@@ -0,0 +1,84 @@
+"""Distribution step — rsync exported scenery to mirror servers.
+
+Equivalent to the legacy sync.sh script.
+This step shells out to rsync since it's the right tool for efficient
+network file transfer. SSH configuration (host, user, key) is read from
+~/.ssh/config as usual.
+"""
+
+import logging
+import os
+import subprocess
+import shlex
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+logger = logging.getLogger(__name__)
+
+
+def rsync_to_target(scenery_dir: str, target: str, items: list[str]) -> int:
+    """Rsync specific items to a target.
+
+    Args:
+        scenery_dir: Local scenery directory (source).
+        target: rsync target URI (e.g., "host:/path/").
+        items: List of file/directory names to sync.
+
+    Returns:
+        rsync exit code.
+    """
+    cmd = [
+        "rsync", "-av",
+        "--no-owner", "--no-group", "--no-perms",
+        "--delete", "--copy-links", "--omit-dir-times",
+    ] + items + [target]
+
+    logger.info("rsync to %s: %s", target, " ".join(items))
+    result = subprocess.run(cmd, cwd=scenery_dir, capture_output=True, text=True)
+
+    if result.returncode != 0:
+        logger.error("rsync to %s failed: %s", target, result.stderr[:500])
+    else:
+        logger.info("rsync to %s complete.", target)
+
+    return result.returncode
+
+
+def run(fg_scenery: str, sync_targets: list[str], cdn_invalidate_cmd: str = ""):
+    """Distribute scenery to all mirror targets.
+
+    Args:
+        fg_scenery: Local scenery directory.
+        sync_targets: List of rsync target URIs.
+        cdn_invalidate_cmd: Optional shell command to invalidate CDN.
+    """
+    if not sync_targets:
+        logger.info("No sync targets configured. Skipping distribution.")
+        return
+
+    scenery_items = ["Objects", "Models", "Airports", ".dirindex", "Airports_archive.tgz"]
+    tarball = "SharedModels.txz"
+
+    with ThreadPoolExecutor(max_workers=len(sync_targets) * 2) as executor:
+        futures = []
+
+        for target in sync_targets:
+            # Main scenery sync
+            futures.append(
+                executor.submit(rsync_to_target, fg_scenery, target, scenery_items)
+            )
+
+            # SharedModels tarball to parent directory
+            tarball_path = os.path.join(fg_scenery, tarball)
+            if os.path.isfile(tarball_path):
+                parent_target = target.rsplit("/", 2)[0] + "/"
+                futures.append(
+                    executor.submit(rsync_to_target, fg_scenery, parent_target, [tarball])
+                )
+
+        for future in as_completed(futures):
+            future.result()
+
+    # Invalidate CDN if configured
+    if cdn_invalidate_cmd:
+        logger.info("Invalidating CDN: %s", cdn_invalidate_cmd)
+        subprocess.run(shlex.split(cdn_invalidate_cmd), check=False)

fgsmdb/steps/sanitize.py

@@ -0,0 +1,32 @@
+"""Sanitize step — fix data integrity issues in the database.
+
+Equivalent to the legacy sanitize.sh script.
+"""
+
+import logging
+from ..db import Database
+
+logger = logging.getLogger(__name__)
+
+
+def run(db: Database):
+    """Fix tile numbers and elevation offsets in the database."""
+    logger.info("Running sanitize...")
+
+    statements = [
+        # Update object tile numbers where they don't match geometry
+        """UPDATE fgs_objects SET ob_tile = fn_GetTileNumber(wkb_geometry)
+           WHERE ob_tile != fn_GetTileNumber(wkb_geometry) OR ob_tile IS NULL""",
+        # Update sign tile numbers where invalid
+        """UPDATE fgs_signs SET si_tile = fn_GetTileNumber(wkb_geometry)
+           WHERE si_tile < 1 OR si_tile IS NULL""",
+        # Nullify zero elevation offsets
+        """UPDATE fgs_objects SET ob_elevoffset = NULL
+           WHERE ob_elevoffset = 0""",
+    ]
+
+    for stmt in statements:
+        logger.debug("Executing: %s", stmt.strip()[:80])
+        db.execute(stmt)
+
+    logger.info("Sanitize complete.")

fgsmdb/steps/sync.py

@@ -0,0 +1,150 @@
+"""Sync step — synchronize exported scenery to the local scenery tree.
+
+Equivalent to the legacy upload.sh + LocalSync.py scripts.
+Uses .dirindex-based diffing to copy only changed files.
+"""
+
+import hashlib
+import logging
+import os
+import shutil
+
+logger = logging.getLogger(__name__)
+
+DIRINDEX_FILENAME = ".dirindex"
+
+
+def sha1_of_file(path: str) -> str:
+    """Compute SHA1 hash of a file."""
+    h = hashlib.sha1()
+    try:
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                h.update(chunk)
+    except OSError:
+        pass
+    return h.hexdigest()
+
+
+class DirIndex:
+    """Parser for .dirindex files."""
+
+    def __init__(self, path: str):
+        self.directories: list[dict] = []
+        self.files: list[dict] = []
+        self.version = 0
+        self.path = ""
+
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+
+                tokens = line.split(":")
+                if tokens[0] == "version":
+                    self.version = int(tokens[1])
+                elif tokens[0] == "path":
+                    self.path = tokens[1] if len(tokens) > 1 else ""
+                elif tokens[0] == "d":
+                    self.directories.append({"name": tokens[1], "hash": tokens[2]})
+                elif tokens[0] in ("f", "t"):
+                    self.files.append({
+                        "name": tokens[1],
+                        "hash": tokens[2],
+                        "size": tokens[3] if len(tokens) > 3 else "0",
+                    })
+
+    def get_file_by_name(self, name: str) -> dict | None:
+        """Find a file entry by name."""
+        for f in self.files:
+            if f["name"] == name:
+                return f
+        return None
+
+
+def sync_path(src: str, dst: str, expected_hash: str | None = None) -> str | None:
+    """Sync a directory from src to dst using .dirindex-based diffing.
+
+    Only copies files whose hashes differ between source and destination.
+    Removes files present in destination but not in source.
+
+    Args:
+        src: Source directory path.
+        dst: Destination directory path.
+        expected_hash: Expected hash of the source .dirindex (for validation).
+
+    Returns:
+        SHA1 hash of the source .dirindex file.
+    """
+    src_dirindex_path = os.path.join(src, DIRINDEX_FILENAME)
+    if not os.path.isfile(src_dirindex_path):
+        logger.debug("No .dirindex at %s, skipping.", src)
+        return None
+
+    src_index = DirIndex(src_dirindex_path)
+
+    # Recurse into subdirectories first
+    for sub_dir in src_index.directories:
+        sync_path(
+            os.path.join(src, sub_dir["name"]),
+            os.path.join(dst, sub_dir["name"]),
+            sub_dir["hash"],
+        )
+
+    # Compute hash if not provided
+    if expected_hash is None:
+        expected_hash = sha1_of_file(src_dirindex_path)
+
+    dst_dirindex_path = os.path.join(dst, DIRINDEX_FILENAME)
+
+    if not os.path.isfile(dst_dirindex_path):
+        # No destination .dirindex — copy everything
+        os.makedirs(dst, exist_ok=True)
+        for file_entry in src_index.files:
+            _copy_file(file_entry["name"], src, dst)
+            logger.debug("copy (new) %s : %s -> %s", file_entry["name"], src, dst)
+        _copy_file(DIRINDEX_FILENAME, src, dst)
+    else:
+        # Have destination .dirindex — do incremental sync
+        dst_index = DirIndex(dst_dirindex_path)
+
+        for src_file in src_index.files:
+            dst_file = dst_index.get_file_by_name(src_file["name"])
+            if dst_file is None or dst_file["hash"] != src_file["hash"]:
+                os.makedirs(dst, exist_ok=True)
+                _copy_file(src_file["name"], src, dst)
+                logger.debug("copy (changed) %s : %s -> %s", src_file["name"], src, dst)
+
+        # Update .dirindex if hashes differ
+        if expected_hash != sha1_of_file(dst_dirindex_path):
+            _copy_file(DIRINDEX_FILENAME, src, dst)
+
+        # Remove files that no longer exist in source
+        for dst_file in dst_index.files:
+            if src_index.get_file_by_name(dst_file["name"]) is None:
+                dst_file_path = os.path.join(dst, dst_file["name"])
+                if os.path.exists(dst_file_path):
+                    os.remove(dst_file_path)
+                    logger.debug("delete %s", dst_file_path)
+
+    return expected_hash
+
+
+def _copy_file(name: str, src_dir: str, dst_dir: str):
+    """Copy a single file from src_dir to dst_dir."""
+    src_path = os.path.join(src_dir, name)
+    dst_path = os.path.join(dst_dir, name)
+    shutil.copy2(src_path, dst_path)
+
+
+def run(fg_scenery_export: str, fg_scenery: str):
+    """Sync the export directory into the scenery directory.
+
+    Args:
+        fg_scenery_export: Source export directory.
+        fg_scenery: Destination scenery directory.
+    """
+    logger.info("Syncing export to scenery: %s -> %s", fg_scenery_export, fg_scenery)
+    sync_path(fg_scenery_export, fg_scenery)
+    logger.info("Sync complete.")

fgsmdb-0.1.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: fgsmdb
+Version: 0.1.0
+Summary: FlightGear Scenemodels Database Tools
+Author-email: Torsten Dreyer <torsten@t3r.de>
+License-Expression: GPL-2.0-only
+Project-URL: Homepage, https://github.com/FlightGear/scenemodels
+Project-URL: Repository, https://github.com/FlightGear/scenemodels
+Project-URL: Issues, https://github.com/FlightGear/scenemodels/issues
+Keywords: flightgear,scenery,terrasync,export
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Games/Entertainment :: Simulation
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: psycopg2-binary>=2.9
+Requires-Dist: python-dotenv>=1.0
+
+# fgsmdb — FlightGear Scenemodels Database Tools
+
+Command-line tools for the FlightGear scenemodels database.
+
+## Installation
+
+```bash
+pip install fgsmdb
+```
+
+Or install from source:
+
+```bash
+cd export-legacy/fgsmdb
+pip install .
+```
+
+For development:
+
+```bash
+pip install -e .
+```
+
+## Configuration
+
+Copy `.env.example` to `.env` and fill in your values:
+
+```bash
+cp .env.example .env
+```
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `PGURI` | PostgreSQL connection URI (required) | — |
+| `FGELEV_PATH` | Path to the fgelev binary | `/usr/local/bin/fgelev` |
+| `FG_ROOT` | FlightGear data root (needed by fgelev) | `/app/data` |
+| `FG_SCENERY` | Local scenery path (Terrain, Airports, etc.) | `./fgscenery` |
+| `FG_SCENERY_EXPORT` | Working directory for exported files | `./export` |
+| `GROUNDNETS_PATH` | Path to groundnets git repository | `./groundnets` |
+| `MIN_UPDATES` | Minimum pending updates to trigger export | `1` |
+| `SYNC_TARGETS` | Comma-separated rsync target URIs | — |
+| `CDN_INVALIDATE_CMD` | Shell command to run after distribution | — |
+
+For remote distribution, rsync uses your `~/.ssh/config` for host aliases,
+usernames, and identity files.
+
+## Usage
+
+```bash
+# Run the export pipeline
+python3 -m fgsmdb export
+
+# Or use the installed command
+fgsmdb export
+```
+
+### Export Command
+
+Exports the scenemodels database to a TerraSync-compatible file tree.
+
+```bash
+# Full pipeline (all steps except distribute)
+python3 -m fgsmdb export
+
+# With a specific .env file
+python3 -m fgsmdb export --env-file /path/to/.env
+
+# Run specific steps only
+python3 -m fgsmdb export --steps sanitize,export,mkdiridx
+
+# Include distribution to mirrors
+python3 -m fgsmdb export --steps sanitize,gndelev,export,mkdiridx,upload,distribute
+
+# Skip the pending-updates threshold check
+python3 -m fgsmdb export --skip-check
+
+# Override the minimum updates threshold
+python3 -m fgsmdb export --min-updates 10
+
+# Enable verbose (debug) logging
+python3 -m fgsmdb export -v
+```
+
+### Pipeline Steps
+
+| Step | Description |
+|------|-------------|
+| `sanitize` | Fix data integrity: update tile numbers, nullify zero elevation offsets |
+| `gndelev` | Compute ground elevation for new objects using fgelev |
+| `export` | Export shared/per-hemisphere models and generate .stg files from the database |
+| `mkdiridx` | Create `.dirindex` files for TerraSync and the Airports archive |
+| `upload` | Sync exported files to the local scenery tree (hash-based, copies only changes) |
+| `distribute` | Rsync scenery to remote mirror servers (parallel) |
+
+By default, all steps except `distribute` are run. Distribution must be explicitly
+requested via `--steps`.
+
+Before any steps execute, the pipeline calls `fn_update_statistics()` and checks
+whether the number of objects pending ground elevation meets the configured threshold.
+Use `--skip-check` to bypass this gate.
+
+### Command Line Options
+
+```
+python3 -m fgsmdb export [-h] [--env-file ENV_FILE] [--steps STEPS] [--all]
+                         [--skip-check] [--min-updates N] [-v]
+
+Options:
+  --env-file ENV_FILE   Path to .env file (default: auto-detect)
+  --steps STEPS         Comma-separated steps: sanitize, gndelev, export,
+                        mkdiridx, upload, distribute
+  --all                 Run all steps (same as default, without distribute)
+  --skip-check          Skip the pending-updates threshold check
+  --min-updates N       Override the MIN_UPDATES threshold from .env
+  -v, --verbose         Enable debug logging
+  -h, --help            Show help message
+```
+
+## Output Structure
+
+```
+fgscenery/
+├── .dirindex
+├── Airports_archive.tgz
+├── SharedModels.txz
+├── Airports/
+│   ├── .dirindex
+│   └── ...
+├── Models/
+│   ├── .dirindex
+│   └── <group_path>/
+├── Objects/
+│   ├── .dirindex
+│   └── <e|w>NNN<n|s>NN/
+│       └── <e|w>NNN<n|s>NN/
+│           ├── <tile>.stg
+│           └── <model_files>
+└── Terrain/
+    └── .dirindex
+```
+
+## License
+
+GNU General Public License v2.0 — see [LICENCE](../LICENCE).

fgsmdb-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+fgsmdb/__init__.py,sha256=d0Xj6sy_mmFEazfAaqFG93uC37JJk5_o-eWDuDhAOws,79
+fgsmdb/__main__.py,sha256=VEts6B7jmw5qF-6jdOIhXcZf9NdNcj-6xWAVYnMBSgw,643
+fgsmdb/config.py,sha256=bviHS-NON-dTplcvb6JvVz91bsb84I047Ca9XBCSpcg,1252
+fgsmdb/db.py,sha256=FP3_Cq4BnH2J-_qVPIYfox1Wzm7uiHhYS3vu960t2wk,2038
+fgsmdb/commands/__init__.py,sha256=R8BQiqA6D0JentyffjXv6ffF6ZyrH25T0AaHogg1SwM,25
+fgsmdb/commands/export.py,sha256=QJaNoo1HhYxy1-50x4QyHy_yhhnjGbcxWxXrjOXCEww,5937
+fgsmdb/steps/__init__.py,sha256=H9UzBR9zBfZalAFewdzh3IZ0lQTa0xDsh19JJrPkv20,31
+fgsmdb/steps/dirindex.py,sha256=UZ3VxVmEIiTVmXilolswlTkmUslmAbZrJAg9VCGJ7dU,5388
+fgsmdb/steps/export_models.py,sha256=Ej08S_ijPAtWo88HmfWM1-qW0xSZRDMpxyM4Frl_OqU,3331
+fgsmdb/steps/export_stg.py,sha256=qf9oH5s8pgl7nH8BhBbcs__PO8V1X4UNKQR-7hp_Y2w,1853
+fgsmdb/steps/gndelev.py,sha256=h3pUQ0rsXsUxX2SOY944PpHN7bRrFFolKCLt4kJqXTQ,3239
+fgsmdb/steps/rsync_distribute.py,sha256=mEPStPO6XeEY7OmgPCrkm0QLsHBHFYi2dvf7kY1qgJ8,2774
+fgsmdb/steps/sanitize.py,sha256=IF0J4pgTxscA03F1U2BJ51hAT0keAkWKmx8o8gFvaPI,1035
+fgsmdb/steps/sync.py,sha256=6PeVfIwL0q96W-VpMwX-7M8Afa5U0elGpOUEBbyEfjo,5100
+fgsmdb-0.1.0.dist-info/METADATA,sha256=-7gLREoGkU9pMozI14E7ry8_qg8rVJ-l_WkPEmxuMHU,5122
+fgsmdb-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+fgsmdb-0.1.0.dist-info/entry_points.txt,sha256=7anFRXVzUAzM8vzeKUKEr4LQjdAy8KaYA4Nnut0Umy8,48
+fgsmdb-0.1.0.dist-info/top_level.txt,sha256=SAnULTGAK0uWFh-Yn9bD-uIz9iR1U_iQUJyCWfF4YKc,7
+fgsmdb-0.1.0.dist-info/RECORD,,

fgsmdb-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fgsmdb-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fgsmdb = fgsmdb.__main__:main

fgsmdb-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fgsmdb