tradedangerous 11.5.3__py3-none-any.whl → 12.0.1__py3-none-any.whl

tradedangerous/plugins/eddblink_plug.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 """
 Import plugin that uses data files from 
 https://elite.tromador.com/ to update the Database.
 """
-from __future__ import annotations
+
 
 from email.utils import parsedate_to_datetime
 from pathlib import Path
-from .. fs import file_line_count
+from ..fs import file_line_count
 from .. import plugins, cache, transfers
 from ..misc import progress as pbar
 from ..plugins import PluginException
@@ -15,13 +17,15 @@ import csv
 import datetime
 import os
 import requests
-import sqlite3
 import typing
 
+from sqlalchemy.orm import Session
+from sqlalchemy import func, delete, select, exists, text
+from ..db import orm_models as SA, lifecycle
 
 if typing.TYPE_CHECKING:
     from typing import Optional
-    from .. tradeenv import TradeEnv
+    from ..tradeenv import TradeEnv
 
 # Constants
 BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
@@ -36,7 +40,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
     if not listings.exists():
         tdenv.NOTE("File not found, aborting: {}", listings)
         return 0
-    
+
     tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
     count = file_line_count(listings)
     if count <= 1:
@@ -45,26 +49,39 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
         else:
             tdenv.DEBUG0("Listings file is empty, nothing to do.")
         return 0
-    
+
     return count + 1  # kfsone: Doesn't the header already make this + 1?
 
 
-def _make_item_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
-    """ helper: retrieve the list of commodities in database. """
+def _make_item_id_lookup(tdenv: TradeEnv, session: Session) -> frozenset[int]:
+    """Helper: retrieve the list of commodities in database."""
     tdenv.DEBUG0("Getting list of commodities...")
-    return frozenset(cols[0] for cols in db.execute("SELECT item_id FROM Item"))
+    rows = session.query(SA.Item.item_id).all()
+    return frozenset(r[0] for r in rows)
 
 
-def _make_station_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
-    """ helper: retrieve the list of station IDs in database. """
+def _make_station_id_lookup(tdenv: TradeEnv, session: Session) -> frozenset[int]:
+    """Helper: retrieve the list of station IDs in database."""
     tdenv.DEBUG0("Getting list of stations...")
-    return frozenset(cols[0] for cols in db.execute("SELECT station_id FROM Station"))
+    rows = session.query(SA.Station.station_id).all()
+    return frozenset(r[0] for r in rows)
 
 
-def _collect_station_modified_times(tdenv: TradeEnv, db: sqlite3.Cursor) -> dict[int, int]:
-    """ helper: build a list of the last modified time for all stations by id. """
+def _collect_station_modified_times(tdenv: TradeEnv, session: Session) -> dict[int, int]:
+    """Helper: build a list of the last modified time for all stations by id (epoch seconds)."""
     tdenv.DEBUG0("Getting last-update times for stations...")
-    return dict(db.execute("SELECT station_id, strftime('%s', MIN(modified)) FROM StationItem GROUP BY station_id"))
+    rows = (
+        session.query(
+            SA.StationItem.station_id,
+            func.min(SA.StationItem.modified),
+        )
+        .group_by(SA.StationItem.station_id)
+        .all()
+    )
+    return {
+        station_id: int(modified.timestamp()) if modified else 0
+        for station_id, modified in rows
+    }
 
 
 class ImportPlugin(plugins.ImportPluginBase):
@@ -91,10 +108,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         'optimize':     "Optimize ('vacuum') database after processing.",
         'solo':         "Don't download crowd-sourced market data. (Implies '-O skipvend', supercedes '-O all', '-O clean', '-O listings'.)",
     }
-    
+
     def __init__(self, tdb, tdenv):
         super().__init__(tdb, tdenv)
-        
+
         self.dataPath = os.environ.get('TD_EDDB') or self.tdenv.tmpDir
         self.categoriesPath = Path("Category.csv")
         self.commoditiesPath = Path("Item.csv")
@@ -112,10 +129,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         self.listingsPath = Path("listings.csv")
         self.liveListingsPath = Path("listings-live.csv")
         self.pricesPath = Path("listings.prices")
-    
+
     def now(self):
         return datetime.datetime.now()
-    
+
     def downloadFile(self, path):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
@@ -124,9 +141,9 @@ class ImportPlugin(plugins.ImportPluginBase):
             localPath = Path(self.tdb.dataPath, path)
         else:
             localPath = Path(self.dataPath, path)
-        
-        url  = BASE_URL + str(path)
-        
+
+        url = BASE_URL + str(path)
+
         self.tdenv.NOTE("Checking for update to '{}'.", path)
         # Use an HTTP Request header to obtain the Last-Modified and Content-Length headers.
         # Also, tell the server to give us the un-compressed length of the file by saying
@@ -137,277 +154,264 @@ class ImportPlugin(plugins.ImportPluginBase):
         except Exception as e:  # pylint: disable=broad-exception-caught
             self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", url, str(e))
             return False
-        
+
         last_modified = response.headers.get("last-modified")
         dump_mod_time = parsedate_to_datetime(last_modified).timestamp()
-        
+
         if Path.exists(localPath):
             local_mod_time = localPath.stat().st_mtime
             if local_mod_time >= dump_mod_time:
                 self.tdenv.DEBUG0("'{}': Dump is not more recent than Local.", path)
                 return False
-        
+
         # The server doesn't know the gzip'd length, and we won't see the gzip'd data,
         # so we want the actual text-only length. Capture it here so we can tell the
         # transfer mechanism how big the file is going to be.
         length = response.headers.get("content-length")
-        
+
         self.tdenv.NOTE("Downloading file '{}'.", path)
         transfers.download(self.tdenv, url, localPath, chunkSize=16384, length=length)
-        
+
         # Change the timestamps on the file so they match the website
         os.utime(localPath, (dump_mod_time, dump_mod_time))
-        
+
         return True
-    
+
     def purgeSystems(self):
         """
         Purges systems from the System table that do not have any stations claiming to be in them.
         Keeps table from becoming too large because of fleet carriers moving to unpopulated systems.
         """
-        db = self.tdb.getDB()
         self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now())
-        
-        db.execute("""
-            DELETE FROM System
-             WHERE NOT EXISTS(SELECT 1 FROM Station WHERE Station.system_id = System.system_id)
-        """)
-        db.commit()
-        
+
+        Session = self.tdb.Session
+        with Session.begin() as session:
+            subq = select(SA.Station.system_id).where(SA.Station.system_id == SA.System.system_id)
+            stmt = delete(SA.System).where(~exists(subq))
+            session.execute(stmt)
+
         self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now())
-    
+
     def importListings(self, listings_file):
         """
-        Updates the market data (AKA the StationItem table) using listings_file
-        Writes directly to database.
+        Updates the market data (StationItem) using `listings_file`.
+
+        Rules:
+          - If a row doesn't exist in DB → insert (copy CSV exactly).
+          - If it exists → update only when CSV.modified > DB.modified.
+          - If CSV.modified <= DB.modified → do nothing (no field changes).
         """
+        from tradedangerous.db.utils import (
+            get_import_batch_size,
+            begin_bulk_mode,
+            end_bulk_mode,
+            get_upsert_fn,
+        )
+
         listings_path = Path(self.dataPath, listings_file).absolute()
-        from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
-        
+        from_live = int(listings_path != Path(self.dataPath, self.listingsPath).absolute())
+
         self.tdenv.NOTE("Checking listings")
         total = _count_listing_entries(self.tdenv, listings_path)
         if not total:
             self.tdenv.NOTE("No listings")
             return
-        
-        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
-        
-        db = self.tdb.getDB()
-        stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
-        stmt_flush_station   = """DELETE from StationItem WHERE station_id = ?"""
-        stmt_add_listing     = """
-            INSERT OR IGNORE INTO StationItem (
-                station_id, item_id, modified, from_live,
-                demand_price, demand_units, demand_level,
-                supply_price, supply_units, supply_level
-            )
-            VALUES (
-                ?, ?, datetime(?, 'unixepoch'), ?,
-                ?, ?, ?,
-                ?, ?, ?
-            )
-        """
-        
-        # Fetch all the items IDS
-        item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
-        station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
-        last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
-        
-        cur_station = None
+
+        self.tdenv.NOTE(
+            "Processing market data from {}: Start time = {}. Live = {}",
+            listings_file, self.now(), bool(from_live)
+        )
+
+        Session = self.tdb.Session
+
+        # Prefetch item/station IDs for early filtering
+        with Session.begin() as session:
+            item_lookup = _make_item_id_lookup(self.tdenv, session)
+            station_lookup = _make_station_id_lookup(self.tdenv, session)
+
         is_debug = self.tdenv.debug > 0
         self.tdenv.DEBUG0("Processing entries...")
-        
-        # Try to find a balance between doing too many commits where we fail
-        # to get any benefits from constructing transactions, and blowing up
-        # the WAL and memory usage by making massive transactions.
-        max_transaction_items, transaction_items = 32 * 1024, 0
-        with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog,\
-              listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
-            cursor = db.cursor()
-            cursor.execute("BEGIN TRANSACTION")
-            
-            for listing in csv.DictReader(fh):
-                prog.increment(1)
-                
-                station_id = int(listing['station_id'])
-                if station_id not in station_lookup:
-                    continue
-                
-                listing_time = int(listing['collected_at'])
-                
-                if station_id != cur_station:
-                    # commit anything from the previous station, get a new cursor
-                    if transaction_items >= max_transaction_items:
-                        cursor.execute("COMMIT")
-                        transaction_items = 0
-                        cursor.execute("BEGIN TRANSACTION")
-                    cur_station, skip_station = station_id, False
-                    
-                    # Check if listing already exists in DB and needs updated.
-                    last_modified: int = int(last_station_update_times.get(station_id, 0))
-                    if last_modified:
-                        # When the listings.csv data matches the database, update to make from_live == 0.
-                        if listing_time == last_modified and not from_live:
-                            if is_debug:
-                                self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
-                            cursor.execute(stmt_unliven_station, (cur_station,))
-                            transaction_items += 1
-                            skip_station = True
-                            continue
-                        
-                        # Unless the import file data is newer, nothing else needs to be done for this station,
-                        # so the rest of the listings for this station can be skipped.
-                        if listing_time <= last_modified:
-                            skip_station = True
+
+        with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog, \
+             listings_path.open("r", encoding="utf-8", errors="ignore") as fh, \
+             Session() as session:
+
+            token = begin_bulk_mode(session, profile="eddblink", phase="incremental")
+            try:
+                commit_batch = get_import_batch_size(session, profile="eddblink")
+                execute_batch = commit_batch or 10000  # cap statement size even if single final commit
+
+                # Upsert: keys + guarded fields (including from_live), guarded by 'modified'
+                table = SA.StationItem.__table__
+                key_cols = ("station_id", "item_id")
+                update_cols = (
+                    "demand_price", "demand_units", "demand_level",
+                    "supply_price", "supply_units", "supply_level",
+                    "from_live",
+                )
+                upsert = get_upsert_fn(
+                    session,
+                    table,
+                    key_cols=key_cols,
+                    update_cols=update_cols,
+                    modified_col="modified",
+                    always_update=(),   # IMPORTANT: no unconditional updates
+                )
+
+                batch_rows = []
+                since_commit = 0
+
+                for listing in csv.DictReader(fh):
+                    prog.increment(1)
+                    try:
+                        station_id = int(listing["station_id"])
+                        if station_id not in station_lookup:
                             continue
-                        
-                        # The data from the import file is newer, so we need to delete the old data for this station.
-                        if is_debug:
-                            self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
-                        cursor.execute(stmt_flush_station, (cur_station,))
-                        transaction_items += 1
-                        last_station_update_times[station_id] = listing_time
-                
-                # station skip lasts until we change station id.
-                if skip_station:
-                    continue
-                
-                # Since this station is not being skipped, get the data and prepare for insertion into the DB.
-                item_id = int(listing['commodity_id'])
-                # listings.csv includes rare items, which we are ignoring.
-                if item_id not in item_lookup:
-                    continue
-                
-                demand_price = int(listing['sell_price'])
-                demand_units = int(listing['demand'])
-                demand_level = int(listing.get('demand_bracket') or '-1')
-                supply_price = int(listing['buy_price'])
-                supply_units = int(listing['supply'])
-                supply_level = int(listing.get('supply_bracket') or '-1')
-                
-                if is_debug:
-                    self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
-                cursor.execute(stmt_add_listing, (
-                        station_id, item_id, listing_time, from_live,
-                        demand_price, demand_units, demand_level,
-                        supply_price, supply_units, supply_level,
-                ))
-                transaction_items += 1
-        
-        # These will take a little while, which has four steps, so we'll make it a counter.
+
+                        item_id = int(listing["commodity_id"])
+                        if item_id not in item_lookup:
+                            continue  # skip rare items (not in Item table)
+
+                        listing_time = int(listing["collected_at"])
+                        dt_listing_time = datetime.datetime.utcfromtimestamp(listing_time)
+
+                        row = {
+                            "station_id": station_id,
+                            "item_id": item_id,
+                            "modified": dt_listing_time,   # guard column
+                            "from_live": from_live,        # copied exactly when updating/inserting
+                            "demand_price": int(listing["sell_price"]),
+                            "demand_units": int(listing["demand"]),
+                            "demand_level": int(listing.get("demand_bracket") or "-1"),
+                            "supply_price": int(listing["buy_price"]),
+                            "supply_units": int(listing["supply"]),
+                            "supply_level": int(listing.get("supply_bracket") or "-1"),
+                        }
+                        batch_rows.append(row)
+                        since_commit += 1
+
+                        if len(batch_rows) >= execute_batch:
+                            upsert(batch_rows)
+                            batch_rows.clear()
+
+                        if commit_batch and since_commit >= commit_batch:
+                            session.commit()
+                            since_commit = 0
+
+                    except Exception as e:  # pylint: disable=broad-exception-caught
+                        self.tdenv.WARN("Bad listing row (skipped): {}  error: {}", listing, e)
+                        continue
+
+                if batch_rows:
+                    upsert(batch_rows)
+                    batch_rows.clear()
+
+                session.commit()
+
+            finally:
+                end_bulk_mode(session, token)
+
         with pbar.Progress(1, 40, prefix="Saving"):
-            # Do a final commit to be sure
-            cursor.execute("COMMIT")
-        
+            pass
+
         if self.getOption("optimize"):
             with pbar.Progress(1, 40, prefix="Optimizing"):
-                db.execute("VACUUM")
-        
-        self.tdb.close()
-        
+                if self.tdb.engine.dialect.name == "sqlite":
+                    with Session.begin() as session:
+                        session.execute(text("VACUUM"))
+
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
-    
+
+
     def run(self):
+        """
+        EDDN/EDDB link importer.
+
+        Refactored DB flow:
+          - No dialect-specific logic in the plugin.
+          - Preflight uses TradeDB.reloadCache() (which centralizes sanity via lifecycle.ensure_fresh_db).
+          - For '--clean' → do a single full rebuild with the RareItem dance.
+          - Otherwise, if static CSVs changed → incrementally import only those tables (no drop/recreate).
+          - Listings import and .prices regeneration unchanged.
+        """
+        import os
+        import time
+        from pathlib import Path
+        from tradedangerous import cache
+        # bulk-mode helpers for the incremental static import session
+        from tradedangerous.db.utils import begin_bulk_mode, end_bulk_mode
+
         self.tdenv.ignoreUnknown = True
-        
-        # Create the /eddb folder for downloading the source files if it doesn't exist.
-        try:
-            Path(str(self.dataPath)).mkdir()
-        except FileExistsError:
-            pass
-        
-        # Run 'listings' by default:
-        # If no options, or if only 'force', and/or 'skipvend',
-        # have been passed, enable 'listings'.
+        self.tdb.dataPath.mkdir(parents=True, exist_ok=True)
+
+        # Enable 'listings' by default unless other explicit options are present
         default = True
         for option in self.options:
             if option not in ('force', 'skipvend', 'purge'):
                 default = False
         if default:
             self.options["listings"] = True
-        
-        # We can probably safely assume that the plugin
-        # has never been run if the db file doesn't exist.
-        if not self.tdb.dbPath.exists():
-            self.options["clean"] = True
-        
+
+        # -----------------------------
+        # Optional CLEAN: prepare inputs
+        # -----------------------------
         if self.getOption("clean"):
-            # Rebuild the tables from scratch. Must be done on first run of plugin.
-            # Can be done at anytime with the "clean" option.
+            # Remove CSVs so downloads become the new source of truth
             for name in [
-                "Category",
-                "Item",
-                "RareItem",
-                "Ship",
-                "ShipVendor",
-                "Station",
-                "System",
-                "Upgrade",
-                "UpgradeVendor",
-                "FDevShipyard",
-                "FDevOutfitting",
+                "Category", "Item", "RareItem",
+                "Ship", "ShipVendor",
+                "Station", "System",
+                "Upgrade", "UpgradeVendor",
+                "FDevShipyard", "FDevOutfitting",
             ]:
-                file = self.tdb.dataPath / Path(name + ".csv")
+                f = self.tdb.dataPath / f"{name}.csv"
                 try:
-                    os.remove(str(file))
+                    os.remove(str(f))
                 except FileNotFoundError:
                     pass
-            
-            try:
-                os.remove(str(self.tdb.dataPath) + "/TradeDangerous.db")
-            except FileNotFoundError:
-                pass
+
+            # Remove .prices (will be regenerated later)
             try:
-                os.remove(str(self.tdb.dataPath) + "/TradeDangerous.prices")
+                os.remove(str(self.tdb.dataPath / "TradeDangerous.prices"))
             except FileNotFoundError:
                 pass
-            
-            # Because this is a clean run, we need to temporarily rename the RareItem.csv,
-            # otherwise TD will crash trying to insert the rare items to the database,
-            # because there's nothing in the Station table it tries to pull from.
-            ri_path = self.tdb.dataPath / Path("RareItem.csv")
-            rib_path = ri_path.with_suffix(".tmp")
-            if ri_path.exists():
-                if rib_path.exists():
-                    rib_path.unlink()
-                ri_path.rename(rib_path)
-            
-            self.tdb.close()
-            
-            self.tdb.reloadCache()
-            self.tdb.close()
-            
-            # Now it's safe to move RareItems back.
-            if ri_path.exists():
-                ri_path.unlink()
-            if rib_path.exists():
-                rib_path.rename(ri_path)
-            
+
+            # Stash RareItem.csv so a full rebuild doesn't hit FK issues
+            self._ri_path = self.tdb.dataPath / "RareItem.csv"
+            self._rib_path = self._ri_path.with_suffix(".tmp")
+            if self._ri_path.exists():
+                if self._rib_path.exists():
+                    self._rib_path.unlink()
+                self._ri_path.rename(self._rib_path)
+
+            # Full update after downloads
             self.options["all"] = True
             self.options["force"] = True
-        
-        # Select which options will be updated
+
+        # --------------------------------
+        # Option cascade (unchanged logic)
+        # --------------------------------
         if self.getOption("listings"):
             self.options["item"] = True
             self.options["station"] = True
-        
+
         if self.getOption("shipvend"):
             self.options["ship"] = True
             self.options["station"] = True
-        
+
         if self.getOption("upvend"):
             self.options["upgrade"] = True
             self.options["station"] = True
-        
+
         if self.getOption("item"):
             self.options["station"] = True
-        
+
         if self.getOption("rare"):
             self.options["station"] = True
-        
+
         if self.getOption("station"):
             self.options["system"] = True
-        
+
         if self.getOption("all"):
             self.options["item"] = True
             self.options["rare"] = True
@@ -418,73 +422,205 @@ class ImportPlugin(plugins.ImportPluginBase):
             self.options["upgrade"] = True
             self.options["upvend"] = True
             self.options["listings"] = True
-        
+
         if self.getOption("solo"):
             self.options["listings"] = False
             self.options["skipvend"] = True
-        
+
         if self.getOption("skipvend"):
             self.options["shipvend"] = False
             self.options["upvend"] = False
-        
-        # Download required files and update tables.
-        buildCache = False
+
+        # ---------------------------------------------
+        # Downloads — track which static CSVs changed
+        # ---------------------------------------------
+        changed = {
+            "System": False,
+            "Station": False,
+            "Category": False,
+            "Item": False,
+            "RareItem": False,
+            "Ship": False,
+            "ShipVendor": False,
+            "Upgrade": False,
+            "UpgradeVendor": False,
+            "FDevShipyard": False,
+            "FDevOutfitting": False,
+        }
+
+        # EDCD mirrors
         if self.getOption("upgrade"):
             if self.downloadFile(self.upgradesPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlOutfitting, self.FDevOutfittingPath)
-                buildCache = True
-        
+                changed["Upgrade"] = True
+                changed["FDevOutfitting"] = True
+
         if self.getOption("ship"):
             if self.downloadFile(self.shipPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlShipyard, self.FDevShipyardPath)
-                buildCache = True
-        
+                changed["Ship"] = True
+                changed["FDevShipyard"] = True
+
+        # Core static tables
         if self.getOption("rare"):
             if self.downloadFile(self.rareItemPath) or self.getOption("force"):
-                buildCache = True
-        
+                changed["RareItem"] = True
+
         if self.getOption("shipvend"):
             if self.downloadFile(self.shipVendorPath) or self.getOption("force"):
-                buildCache = True
-        
+                changed["ShipVendor"] = True
+
         if self.getOption("upvend"):
             if self.downloadFile(self.upgradeVendorPath) or self.getOption("force"):
-                buildCache = True
-        
+                changed["UpgradeVendor"] = True
+
         if self.getOption("system"):
             if self.downloadFile(self.sysPath) or self.getOption("force"):
-                buildCache = True
-        
+                changed["System"] = True
+
         if self.getOption("station"):
             if self.downloadFile(self.stationsPath) or self.getOption("force"):
-                buildCache = True
-        
+                changed["Station"] = True
+
         if self.getOption("item"):
             if self.downloadFile(self.commoditiesPath) or self.getOption("force"):
                 self.downloadFile(self.categoriesPath)
-                buildCache = True
-        
-        # Remake the .db files with the updated info.
-        if buildCache:
-            self.tdb.close()
+                changed["Item"] = True
+                changed["Category"] = True
+
+        # -------------------------------------------------------------
+        # Preflight sanity (user-visible): make the pause explicit
+        # -------------------------------------------------------------
+        ri_path = getattr(self, "_ri_path", self.tdb.dataPath / "RareItem.csv")
+        rib_path = getattr(self, "_rib_path", ri_path.with_suffix(".tmp"))
+        rareitem_stashed = False
+        self.tdenv.NOTE("Preflight: verifying database (this can take a while on first run)...")
+        t0 = time.monotonic()
+        try:
+            if ri_path.exists():
+                if not rib_path.exists() and not self.getOption("clean"):
+                    ri_path.rename(rib_path)
+                    rareitem_stashed = True
+
+            # This may no-op or may call buildCache() internally
             self.tdb.reloadCache()
+        finally:
+            if rib_path.exists() and (self.getOption("clean") or rareitem_stashed):
+                if ri_path.exists():
+                    ri_path.unlink()
+                rib_path.rename(ri_path)
+            t1 = time.monotonic()
+            self.tdenv.NOTE("Preflight complete in {:.1f}s.", (t1 - t0))
+
+        # -----------------------------------------------------
+        # Rebuild or Incremental Import?
+        # -----------------------------------------------------
+        if self.getOption("clean"):
+            self.tdenv.NOTE("Performing full rebuild...")
+            self.tdb.close()
+            cache.buildCache(self.tdb, self.tdenv)
             self.tdb.close()
-        
+            self.tdenv.NOTE("Full rebuild complete.")
+        else:
+            # Incremental import of only changed tables (no schema drop)
+            IMPORT_ORDER = [
+                "System",
+                "Station",
+                "Category",
+                "Item",
+                "RareItem",
+                "Ship",
+                "ShipVendor",
+                "Upgrade",
+                "UpgradeVendor",
+                "FDevShipyard",
+                "FDevOutfitting",
+            ]
+
+            any_changed = any(changed.values())
+            if any_changed:
+                self.tdenv.NOTE("Incremental import starting ({} tables changed).", sum(1 for v in changed.values() if v))
+                with self.tdb.Session() as session:
+                    token = begin_bulk_mode(session, profile="eddblink", phase="incremental")
+                    try:
+                        for table_name in IMPORT_ORDER:
+                            if not changed.get(table_name):
+                                continue
+                            import_path = (self.tdb.dataPath / f"{table_name}.csv").resolve()
+                            try:
+                                # Determine a cheap per-table total (header-aware) for display only.
+                                try:
+                                    total = max(file_line_count(import_path) - 1, 0)
+                                except Exception:
+                                    total = 0
+
+                                prefix = f"Processing {table_name}"
+                                # Mirror listings-style progress: single-line if TTY, periodic otherwise.
+                                with pbar.Progress(total or 1, 40, prefix=prefix, style=pbar.LongRunningCountBar) as prog:
+
+                                    def _cb(stats=None, **kwargs):
+                                        """
+                                        Liberal progress callback used by cache.processImportFile.
+                                        Accepts either:
+                                          - int                    → increment by that many rows
+                                          - dict with keys inc/rows/count → increment by that value
+                                          - anything else          → default increment of 1
+                                        """
+                                        inc = 1
+                                        if isinstance(stats, int):
+                                            inc = max(int(stats), 1)
+                                        elif isinstance(stats, dict):
+                                            for k in ("inc", "rows", "count"):
+                                                if k in stats:
+                                                    try:
+                                                        inc = max(int(stats[k]), 1)
+                                                        break
+                                                    except Exception:
+                                                        pass
+                                        prog.increment(inc)
+
+                                    cache.processImportFile(
+                                        self.tdenv,
+                                        session,
+                                        import_path,
+                                        table_name,
+                                        line_callback=_cb,
+                                        call_args={"table": table_name, "total": total},
+                                    )
+
+                                session.commit()
+                                self.tdenv.DEBUG0("Incremental import OK: {} ({})", table_name, import_path)
+
+                            except FileNotFoundError:
+                                self.tdenv.NOTE("{} missing; skipped incremental import ({})", table_name, import_path)
+                            except StopIteration:
+                                self.tdenv.NOTE("{} exists but is empty; skipped incremental import ({})", table_name, import_path)
+                            except Exception as e:
+                                self.tdenv.WARN("Incremental import failed for {}: {} ({})", table_name, e, import_path)
+                                session.rollback()
+                                self.tdenv.NOTE("Escalating to full rebuild due to import failure.")
+                                self.tdb.close()
+                                cache.buildCache(self.tdb, self.tdenv)
+                                self.tdb.close()
+                                break
+                    finally:
+                        end_bulk_mode(session, token)
+                self.tdenv.NOTE("Incremental import finished.")
+
+
         if self.getOption("purge"):
             self.purgeSystems()
-            self.tdb.close()
-        
+
+        # Listings import (prices)
         if self.getOption("listings"):
             if self.downloadFile(self.listingsPath) or self.getOption("force"):
                 self.importListings(self.listingsPath)
             if self.downloadFile(self.liveListingsPath) or self.getOption("force"):
                 self.importListings(self.liveListingsPath)
-        
+
         if self.getOption("listings"):
             self.tdenv.NOTE("Regenerating .prices file.")
             cache.regeneratePricesFile(self.tdb, self.tdenv)
-        
+
         self.tdenv.NOTE("Import completed.")
-        
-        # TD doesn't need to do anything, tell it to just quit.
         return False