tradedangerous 12.7.6__py3-none-any.whl

tradedangerous/plugins/spansh_plug.py

@@ -0,0 +1,2647 @@
+# tradedangerous/plugins/spansh_plug.py
+# -----------------------------------------------------------------------------
+# Spansh Import Plugin (new, defragmented)
+#
+# Behavioural contract:
+# - Optimised for modify/update (churn-safe via service timestamps)
+# - Streaming reader for huge top-level JSON array
+# - Options: -O url=… | -O file=… (mutually exclusive), -O maxage=<float days>
+# - JSON/intermediate in tmp/, CSV & .prices in data/
+# - Warnings gated by verbosity; low-verbosity uses single-line progress
+# - After import: export CSVs (incl. RareItem) and regenerate TradeDangerous.prices
+# - Returns True from finish() to stop default flow
+#
+# DB/dialect specifics live in tradedangerous.db.utils (parse_ts, batch sizing, etc.)
+# -----------------------------------------------------------------------------
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta, timezone
+from email.utils import parsedate_to_datetime
+from importlib.resources import files as implib_files, as_file as implib_as_file
+from pathlib import Path
+import csv
+import json  # used for debug
+import io
+import os
+import shutil
+import sys
+import time
+import traceback
+import typing
+
+# SQLAlchemy
+from sqlalchemy import MetaData, Table, select, insert, update, func, and_, or_, text, UniqueConstraint
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+import ijson  # Used for main stream
+import urllib.request
+
+# Framework modules
+from tradedangerous import plugins, cache, csvexport  # provided by project
+from tradedangerous.cache import processImportFile
+
+# DB helpers (dialect specifics live here)
+from tradedangerous.db import utils as db_utils
+from tradedangerous.db.lifecycle import ensure_fresh_db, reset_db
+from tradedangerous.db.locks import station_advisory_lock
+
+
+if typing.TYPE_CHECKING:
+    from collections.abc import Generator, Iterable, Mapping
+    from typing import Any, Optional
+    from tradedangerous import TradeDB, TradeEnv
+
+
+DEFAULT_URL = "https://downloads.spansh.co.uk/galaxy_stations.json"
+
+
+class ImportPlugin(plugins.ImportPluginBase):
+    """
+    Spansh galaxy dump importer:
+      - Consumes galaxy_stations.json (local file or remote URL)
+      - Updates System, Station, Ship/ShipVendor, Upgrade/UpgradeVendor, Item/StationItem
+      - Respects per-service freshness & optional maxage (days)
+      - Imports RareItem.csv via cache.processImportFile() AFTER systems/stations exist
+      - Exports CSVs (+RareItem) and rebuilds TradeDangerous.prices
+    """
+    
+    pluginInfo = {
+        "name": "spansh",
+        "author": "TD Team",
+        "version": "2.1",
+        "minimum-tb-version": "1.76",
+        "description": "Imports Spansh galaxy dump and refreshes cache artefacts.",
+    }
+    
+    # Correct option contract: dict name -> help text
+    pluginOptions = {
+        "url": "Remote URL to galaxy_stations.json (default if neither url nor file is given)",
+        "file": "Local path to galaxy_stations.json; use '-' to read from stdin",
+        "maxage": "Skip service sections older than <days> (float), evaluated per service",
+        "pricesonly": "Skip import/exports; regenerate TradeDangerous.prices only (for testing).",
+        "force_baseline": "If set, overwrite service blocks to Spansh baseline (from_live=0) and delete any extras.",
+        "skip_stationitems": "Skip exporting StationItem.csv (large). Env: TD_SKIP_STATIONITEM_EXPORT=1",
+        "progress_compact": "Use shorter one-line import status (or set env TD_PROGRESS_COMPACT=1).",
+        # --- EDCD sourcing (hardcoded URLs; can be disabled or overridden) ---
+        "no_edcd": "Disable EDCD preloads (categories, FDev tables) and EDCD rares import.",
+        "edcd_commodity": "Override URL or local path for EDCD commodity.csv.",
+        "edcd_outfitting": "Override URL or local path for EDCD outfitting.csv.",
+        "edcd_shipyard": "Override URL or local path for EDCD shipyard.csv.",
+        "edcd_rares": "Override URL or local path for EDCD rare_commodity.csv.",
+        # --- Extra Debug Options
+        "only_system": "Process only the system with this name or id64; still stream the real file.",
+        "debug_trace": "Emit compact JSONL decision logs to tmp/spansh_trace.jsonl (1 line per decision).",
+    }
+    
+    # Hardcoded EDCD sources (raw GitHub)
+    EDCD_URLS = {
+        "commodity": "https://raw.githubusercontent.com/EDCD/FDevIDs/master/commodity.csv",
+        "outfitting": "https://raw.githubusercontent.com/EDCD/FDevIDs/master/outfitting.csv",
+        "shipyard": "https://raw.githubusercontent.com/EDCD/FDevIDs/master/shipyard.csv",
+        "rares": "https://raw.githubusercontent.com/EDCD/FDevIDs/master/rare_commodity.csv",
+    }
+    
+    tdb: TradeDB
+    tdenv: TradeEnv
+    session: Session | None  # this means you have to check it's been set, though
+    batch_size: int | None
+    
+    # ------------------------------
+    # Construction & plumbing (REPLACEMENT)
+    #
+    def __init__(self, tdb, cmdenv):
+        super().__init__(tdb, cmdenv)
+        self.tdb = tdb
+        self.tdenv = cmdenv
+        self.session = None
+        
+        # Paths (data/tmp) from env/config; fall back defensively
+        self.data_dir = Path(getattr(self.tdenv, "dataDir", getattr(self.tdb, "dataDir", "data"))).resolve()
+        self.tmp_dir = Path(getattr(self.tdenv, "tmpDir", getattr(self.tdb, "tmpDir", "tmp"))).resolve()
+        for p in (self.data_dir, self.tmp_dir):
+            try:
+                p.mkdir(parents=True, exist_ok=True)
+            except Exception as e:
+                raise CleanExit(f"Failed to create directory {p}: {e!r}") from None
+        
+        # Batch size decided AFTER session is opened (see finish())
+        self.batch_size = None
+        
+        # Verbosity gates
+        self._is_tty = sys.stderr.isatty() or sys.stdout.isatty()
+        self._debug_level = int(getattr(self.tdenv, "debug", 0) or 0)  # -v levels
+        self._warn_enabled = bool(getattr(self.tdenv, "warn", None)) or (self._debug_level >= 3)
+        
+        # Progress state
+        self._last_progress_time = 0.0
+        
+        # Station type mapping (existing helper in this module)
+        self._station_type_map = self._build_station_type_map()
+        
+        # Debug trace option
+        self.debug_trace = str(self.getOption("debug_trace") or "0").strip().lower() not in ("0", "", "false", "no")
+        self._trace_fp = None
+    
+    # --------------------------------------
+    # Small tracing helper
+    #
+    def _trace(self, **evt) -> None:
+        """
+        Lightweight debug tracer. Writes one compact JSON line per call
+        into tmp/spansh_trace.jsonl when -O debug_trace=1 is set.
+        Has no side-effects on existing logic if disabled.
+        """
+        if not getattr(self, "debug_trace", False):
+            return
+        try:
+            # lazily open file handle if not yet opened
+            if not hasattr(self, "_trace_fp") or self._trace_fp is None:
+                tmp = getattr(self, "tmp_dir", Path("tmp"))
+                tmp.mkdir(parents=True, exist_ok=True)
+                self._trace_fp = (tmp / "spansh_trace.jsonl").open("a", encoding="utf-8")
+            
+            # sanitize datetimes
+            for k, v in list(evt.items()):
+                if hasattr(v, "isoformat"):
+                    evt[k] = v.isoformat()
+            
+            self._trace_fp.write(json.dumps(evt, ensure_ascii=False) + "\n")
+            self._trace_fp.flush()
+        except Exception:
+            pass  # never break main flow
+    
+    # --- TD shim: seed 'Added' from templates (idempotent) ---
+    def _seed_added_from_templates(self, session) -> None:
+        """
+        Seed the legacy 'Added' table from the packaged CSV:
+            tradedangerous/templates/Added.csv
+        
+        DB-agnostic; uses cache.processImportFile. No reliance on any templatesDir.
+        """
+        # Obtain a Traversable for the packaged resource and materialize to a real path
+        res = implib_files("tradedangerous").joinpath("templates", "Added.csv")
+        with implib_as_file(res) as csv_path:
+            if not csv_path.exists():
+                # Graceful failure so schedulers can retry
+                raise CleanExit(f"Packaged Added.csv not found: {csv_path}")
+            try:
+                processImportFile(
+                    tdenv=self.tdenv,
+                    session=session,
+                    importPath=csv_path,
+                    tableName="Added",
+                )
+            except Exception as e:
+                # Keep diagnostics, but avoid hard process exit
+                self._warn("Seeding 'Added' from templates failed; continuing without it.")
+                self._warn(f"{type(e).__name__}: {e}")
+                traceback.print_exc()
+                raise CleanExit("Failed to seed 'Added' table from templates.") from e  # ^ contradiction?
+    
+    # --------------------------------------
+    # EDCD Import Functions
+    #
+    def _acquire_edcd_files(self) -> dict[str, Path | None]:
+        """
+        Download (or resolve) EDCD CSVs to tmp/ with conditional caching.
+        Honors -O no_edcd=1 and per-file overrides:
+          - edcd_commodity, edcd_outfitting, edcd_shipyard, edcd_rares
+        Each override may be a local path or an http(s) URL.
+        Returns dict: {commodity,outfitting,shipyard,rares} -> Path or None.
+        """
+        
+        def _resolve_one(opt_key: str, default_url: str, basename: str) -> Optional[Path]:
+            override = self.getOption(opt_key)
+            target = self.tmp_dir / f"edcd_{basename}.csv"
+            label = f"EDCD {basename}.csv"
+            
+            # Explicit disable via empty override
+            if override is not None and str(override).strip() == "":
+                return None
+            
+            # Local path override
+            if override and ("://" not in override):
+                p = Path(override)
+                if not p.exists():
+                    cwd = getattr(self.tdenv, "cwDir", None)
+                    if cwd:
+                        p = Path(cwd, override)
+                if p.exists() and p.is_file():
+                    return p.resolve()
+                override = None  # fall back to URL
+            
+            # URL (override or default)
+            url = override or default_url
+            try:
+                return self._download_with_cache(url, target, label=label)
+            except CleanExit:
+                return target if target.exists() else None
+            except Exception:
+                return target if target.exists() else None
+        
+        if self.getOption("no_edcd"):
+            return {"commodity": None, "outfitting": None, "shipyard": None, "rares": None}
+        
+        return {
+            "commodity": _resolve_one("edcd_commodity", self.EDCD_URLS["commodity"], "commodity"),
+            "outfitting": _resolve_one("edcd_outfitting", self.EDCD_URLS["outfitting"], "outfitting"),
+            "shipyard":  _resolve_one("edcd_shipyard",  self.EDCD_URLS["shipyard"],  "shipyard"),
+            "rares":     _resolve_one("edcd_rares",     self.EDCD_URLS["rares"],     "rare_commodity"),
+        }
+    
+    # ---------- EDCD: Categories (add-only) ----------
+    #
+    def _edcd_import_categories_add_only(
+            self,
+            session: Session,
+            tables: dict[str, Table],
+            commodity_csv: Path,
+        ) -> int:
+        """
+        Read EDCD commodity.csv, extract distinct category names, and add any
+        missing Category rows. No updates, no deletes.
+        
+        Deterministic + append-only behaviour:
+          - If Category is empty: seed the TD canonical categories with fixed IDs (1..16).
+          - If Category is non-empty: validate the canonical ID→name mapping; abort if drifted.
+          - Any new categories found in EDCD are appended with IDs > current max ID,
+            in deterministic (case-insensitive) name order.
+            
+            Yes, we shoulda done it alphabetical in the first place, but we didn't, so
+            here we are.
+        
+        Returns: number of rows inserted (seed + appended).
+        """
+        t_cat = tables["Category"]
+        
+        # TD canonical mapping — frozen IDs
+        canonical_by_id: dict[int, str] = {
+            1:  "Metals",
+            2:  "Minerals",
+            3:  "Chemicals",
+            4:  "Foods",
+            5:  "Textiles",
+            6:  "Industrial Materials",
+            7:  "Medicines",
+            8:  "Legal Drugs",
+            9:  "Machinery",
+            10: "Technology",
+            11: "Weapons",
+            12: "Consumer Items",
+            13: "Slavery",
+            14: "Waste",
+            15: "NonMarketable",
+            16: "Salvage",
+        }
+        
+        inserted = 0
+        
+        # Load existing categories
+        rows = session.execute(select(t_cat.c.category_id, t_cat.c.name)).all()
+        existing_by_id: dict[int, str] = {
+            int(cid): (str(name) if name is not None else "")
+            for (cid, name) in rows
+        }
+        
+        # Seed canonical set if empty
+        if not existing_by_id:
+            seed_rows = [
+                {"category_id": cid, "name": name}
+                for cid, name in sorted(canonical_by_id.items(), key=lambda kv: kv[0])
+            ]
+            session.execute(insert(t_cat), seed_rows)
+            inserted += len(seed_rows)
+            existing_by_id = {cid: name for cid, name in canonical_by_id.items()}
+        
+        # Sanity guardrail: detect drift
+        else:
+            for cid, expected_name in canonical_by_id.items():
+                if cid not in existing_by_id:
+                    raise CleanExit(
+                        "Category ID drift detected: "
+                        f"missing canonical category_id={cid} (expected '{expected_name}'). "
+                        "Refusing to proceed."
+                    )
+                actual = (existing_by_id.get(cid) or "").strip()
+                if actual.lower() != expected_name.lower():
+                    raise CleanExit(
+                        "Category ID drift detected: "
+                        f"category_id={cid} expected '{expected_name}' but found '{actual}'. "
+                        "Refusing to proceed."
+                    )
+        
+        existing_lc = {
+            (str(n) or "").strip().lower()
+            for n in existing_by_id.values()
+            if n is not None
+        }
+        
+        # Parse EDCD commodity.csv and collect category spellings (case-insensitive)
+        with open(commodity_csv, "r", encoding="utf-8", newline="") as fh:
+            reader = csv.DictReader(fh)
+            
+            cat_col = None
+            for h in (reader.fieldnames or []):
+                if h and str(h).strip().lower() == "category":
+                    cat_col = h
+                    break
+            if cat_col is None:
+                raise CleanExit(f"EDCD commodity.csv missing 'category' column: {commodity_csv}")
+            
+            # lk -> set(spellings)
+            seen: dict[str, set[str]] = {}
+            
+            for row in reader:
+                raw = row.get(cat_col)
+                if not raw:
+                    continue
+                name = str(raw).strip()
+                if not name:
+                    continue
+                
+                lk = name.lower()
+                if lk in existing_lc:
+                    continue
+                
+                seen.setdefault(lk, set()).add(name)
+        
+        if not seen:
+            return inserted
+        
+        # Deterministic selection of display name per lk
+        def _choose_name(spellings: set[str]) -> str:
+            # stable across rebuilds even if EDCD row order changes
+            return min(spellings, key=lambda s: (s.casefold(), s))
+        
+        new_names: list[str] = [_choose_name(seen[lk]) for lk in sorted(seen.keys())]
+        
+        max_id = max(existing_by_id.keys(), default=0)
+        to_add = []
+        next_id = max_id + 1
+        for nm in new_names:
+            to_add.append({"category_id": next_id, "name": nm})
+            next_id += 1
+        
+        session.execute(insert(t_cat), to_add)
+        inserted += len(to_add)
+        return inserted
+    
+    # ---------- EDCD: FDev tables (direct load) ----------
+    #
+    
+    def _edcd_import_table_direct(self, session: Session, table: Table, csv_path: Path) -> int:
+        """
+        Upsert CSV rows into a table whose columns match CSV headers.
+        Prefers the table's primary key; if absent, falls back to a single-column
+        UNIQUE key (e.g. 'id' in FDev tables). Returns approx rows written.
+        """
+        # --- choose key columns for upsert ---
+        pk_cols = tuple(c.name for c in table.primary_key.columns)
+        key_cols: tuple[str, ...] = pk_cols
+        
+        if not key_cols:
+            # Common case for EDCD FDev tables: UNIQUE(id) but no PK
+            if "id" in table.c:
+                key_cols = ("id",)
+            else:
+                # Try to discover a single-column UNIQUE constraint via reflection
+                try:
+                    uniq_single = []
+                    for cons in getattr(table, "constraints", set()):
+                        if isinstance(cons, UniqueConstraint):
+                            cols = tuple(col.name for col in cons.columns)
+                            if len(cols) == 1:
+                                uniq_single.append(cols[0])
+                    if uniq_single:
+                        key_cols = (uniq_single[0],)
+                except Exception:
+                    pass
+        
+        if not key_cols:
+            raise CleanExit(f"Table {table.name} has neither a primary key nor a single-column UNIQUE key; cannot upsert from EDCD")
+        
+        # --- read CSV ---
+        with open(csv_path, "r", encoding="utf-8", newline="") as fh:
+            reader = csv.DictReader(fh)
+            cols = [c for c in (reader.fieldnames or []) if c in table.c]
+            if not cols:
+                return 0
+            rows = [{k: row.get(k) for k in cols} for row in reader]
+        
+        if not rows:
+            return 0
+        
+        # --- table-specific sanitation (fixes ck_fdo_mount / ck_fdo_guidance) ---
+        if table.name == "FDevOutfitting":
+            allowed_mount = {"Fixed", "Gimballed", "Turreted"}
+            allowed_guid  = {"Dumbfire", "Seeker", "Swarm"}
+            
+            def _norm(val, allowed):
+                if val is None:
+                    return None
+                s = str(val).strip()
+                if not s or s not in allowed:
+                    return None
+                return s
+            
+            for r in rows:
+                if "mount" in r:
+                    r["mount"] = _norm(r["mount"], allowed_mount)
+                if "guidance" in r:
+                    r["guidance"] = _norm(r["guidance"], allowed_guid)
+        
+        # --- perform upsert using chosen key columns ---
+        upd_cols = tuple(c for c in cols if c not in key_cols)
+        
+        if db_utils.is_sqlite(session):
+            db_utils.sqlite_upsert_simple(session, table, rows=rows, key_cols=key_cols, update_cols=upd_cols)
+            return len(rows)
+        
+        if db_utils.is_mysql(session):
+            db_utils.mysql_upsert_simple(session, table, rows=rows, key_cols=key_cols, update_cols=upd_cols)
+            return len(rows)
+        
+        # Generic backend (read-then-insert/update)
+        for r in rows:
+            cond = and_(*[getattr(table.c, k) == r[k] for k in key_cols])
+            ext = session.execute(select(*[getattr(table.c, k) for k in key_cols]).where(cond)).first()
+            if ext is None:
+                session.execute(insert(table).values(**r))
+            elif upd_cols:
+                session.execute(update(table).where(cond).values(**{k: r[k] for k in upd_cols}))
+        return len(rows)
+    
+    def _edcd_import_fdev_catalogs(self, session: Session, tables: dict[str, Table], *, outfitting_csv: Path, shipyard_csv: Path) -> tuple[int, int]:
+        u = self._edcd_import_table_direct(session, tables["FDevOutfitting"], outfitting_csv)
+        s = self._edcd_import_table_direct(session, tables["FDevShipyard"],   shipyard_csv)
+        return (u, s)
+    
+    # --------------------------------------
+    # Comparison Helpers
+    #
+    def _apply_vendor_block_per_rules(
+        self,
+        t_vendor: Table,
+        station_id: int,
+        ids: Iterable[int],
+        ts_sp: datetime,
+        *,
+        id_col: str,
+    ) -> tuple[int, int, int]:
+        """
+        Per-row rule for ShipVendor / UpgradeVendor:
+          - If db.modified > ts_sp: leave row.
+          - If db.modified == ts_sp: no-op.
+          - If db.modified <  ts_sp: set modified = ts_sp.
+        Deletions:
+          - Remove rows missing in JSON if (db.modified <= ts_sp).
+        Returns (insert_count, update_count, delete_count).
+        """
+        keep_ids = {int(x) for x in ids if x is not None}
+        inserts = updates = deletes = 0
+        
+        # --- INSERT missing (batch) ---
+        if keep_ids:
+            # Find which of keep_ids are missing
+            existing_ids = {
+                int(r[0]) for r in self.session.execute(
+                    select(getattr(t_vendor.c, id_col)).where(
+                        and_(t_vendor.c.station_id == station_id,
+                             getattr(t_vendor.c, id_col).in_(keep_ids))
+                    )
+                ).all()
+            }
+            to_insert = keep_ids - existing_ids
+            if to_insert:
+                self.session.execute(
+                    insert(t_vendor),
+                    [{id_col: vid, "station_id": station_id, "modified": ts_sp} for vid in to_insert]
+                )
+                inserts = len(to_insert)
+        
+        # --- UPDATE only those with modified < ts_sp (batch) ---
+        if keep_ids:
+            res = self.session.execute(
+                update(t_vendor)
+                .where(
+                    and_(
+                        t_vendor.c.station_id == station_id,
+                        getattr(t_vendor.c, id_col).in_(keep_ids),
+                        or_(t_vendor.c.modified.is_(None), t_vendor.c.modified < ts_sp),
+                    )
+                )
+                .values(modified=ts_sp)
+            )
+            # rowcount includes both existing rows (not inserts) whose modified was < ts_sp
+            updates = int(res.rowcount or 0)
+        
+        # --- DELETE rows NOT in keep_ids, but only if <= ts_sp (single statement) ---
+        res = self.session.execute(
+            t_vendor.delete().where(
+                and_(
+                    t_vendor.c.station_id == station_id,
+                    ~getattr(t_vendor.c, id_col).in_(keep_ids) if keep_ids else True,
+                    or_(t_vendor.c.modified.is_(None), t_vendor.c.modified <= ts_sp),
+                )
+            )
+        )
+        deletes = int(res.rowcount or 0)
+        
+        return inserts, updates, deletes
+    
+    def _sync_vendor_block_fast(
+            self,
+            tables: dict[str, Table],
+            *,
+            station_id: int,
+            entries: list[dict[str, Any]],
+            ts_sp: Optional[datetime],
+            kind: str,  # "ship" or "module"
+        ) -> tuple[int, int]:
+        """
+        Fast, set-based vendor sync for a single station and one service (shipyard/outfitting).
+        
+        Returns: (number_of_inserts_or_updates_on_vendor_links, deletions_count).
+        """
+        # Ensure we never write NULL into NOT NULL 'modified' columns.
+        ts_eff = (ts_sp or datetime.utcnow().replace(microsecond=0))
+        
+        if kind == "ship":
+            t_master = tables["Ship"]
+            t_vendor = tables["ShipVendor"]
+            id_key = "shipId"
+            id_col = "ship_id"
+            master_rows = []
+            keep_ids: set[int] = set()
+            for e in entries:
+                if not isinstance(e, dict):
+                    continue
+                ship_id = e.get(id_key)
+                name = e.get("name")
+                if ship_id is None or name is None:
+                    continue
+                keep_ids.add(int(ship_id))
+                master_rows.append({"ship_id": ship_id, "name": name})
+        
+        elif kind == "module":
+            t_master = tables["Upgrade"]
+            t_vendor = tables["UpgradeVendor"]
+            id_key = "moduleId"
+            id_col = "upgrade_id"
+            master_rows = []
+            keep_ids = set()
+            for e in entries:
+                if not isinstance(e, dict):
+                    continue
+                up_id = e.get(id_key)
+                name = e.get("name")
+                if up_id is None or name is None:
+                    continue
+                keep_ids.add(int(up_id))
+                master_rows.append({
+                    "upgrade_id": up_id,
+                    "name": name,
+                    "class": e.get("class"),
+                    "rating": e.get("rating"),
+                    "ship": e.get("ship"),
+                })
+        else:
+            raise CleanExit(f"_sync_vendor_block_fast: unknown kind={kind!r}")
+        
+        # 1) Ensure master rows exist (simple upsert, no timestamp guards).
+        if master_rows:
+            key_name = list(master_rows[0].keys())[0]
+            update_cols = tuple(k for k in master_rows[0].keys() if k != key_name)
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_simple(
+                    self.session, t_master, rows=master_rows,
+                    key_cols=(key_name,),
+                    update_cols=update_cols,
+                )
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_simple(
+                    self.session, t_master, rows=master_rows,
+                    key_cols=(key_name,),
+                    update_cols=update_cols,
+                )
+            else:
+                for r in master_rows:
+                    cond = bool(getattr(t_master.c, key_name) == r[key_name])
+                    exists = self.session.execute(select(getattr(t_master.c, key_name)).where(cond)).first()
+                    if exists is None:
+                        self.session.execute(insert(t_master).values(**r))
+                    else:
+                        upd = {k: v for k, v in r.items() if k != key_name}
+                        if upd:
+                            self.session.execute(update(t_master).where(cond).values(**upd))
+        
+        # 2) Link rows with timestamp guard for vendor tables.
+        wrote = 0
+        delc = 0
+        if keep_ids:
+            existing = {
+                int(r[0]): (r[1] or None)
+                for r in self.session.execute(
+                    select(getattr(t_vendor.c, id_col), t_vendor.c.modified).where(
+                        and_(t_vendor.c.station_id == station_id, getattr(t_vendor.c, id_col).in_(keep_ids))
+                    )
+                ).all()
+            }
+            to_insert = keep_ids - set(existing.keys())
+            to_update = {
+                vid for vid, mod in existing.items()
+                if (mod is None) or (ts_eff > mod)
+            }
+            wrote = len(to_insert) + len(to_update)
+            
+            vendor_rows = [{id_col: vid, "station_id": station_id, "modified": ts_eff} for vid in keep_ids]
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_modified(
+                    self.session, t_vendor, rows=vendor_rows,
+                    key_cols=(id_col, "station_id"),
+                    modified_col="modified",
+                    update_cols=(),
+                )
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_modified(
+                    self.session, t_vendor, rows=vendor_rows,
+                    key_cols=(id_col, "station_id"),
+                    modified_col="modified",
+                    update_cols=(),
+                )
+            else:
+                for r in vendor_rows:
+                    cond = and_(getattr(t_vendor.c, id_col) == r[id_col], t_vendor.c.station_id == station_id)
+                    cur = self.session.execute(select(t_vendor.c.modified).where(cond)).first()
+                    if cur is None:
+                        self.session.execute(insert(t_vendor).values(**r))
+                    else:
+                        mod = cur[0]
+                        if (mod is None) or (ts_eff > mod):
+                            self.session.execute(update(t_vendor).where(cond).values(modified=ts_eff))
+        
+        return wrote, delc
+    
+    def _cleanup_absent_stations(self, tables: dict[str, Table], present_station_ids: set[int], json_ts: datetime) -> tuple[int, int, int]:
+        """
+        After streaming, delete baseline rows for stations absent from the JSON
+        if the JSON timestamp is >= row.modified. Never delete newer-than-JSON rows.
+        Returns (market_del, outfit_del, ship_del) counts.
+        """
+        t_si, t_uv, t_sv, t_st = tables["StationItem"], tables["UpgradeVendor"], tables["ShipVendor"], tables["Station"]
+        
+        # All station ids in DB
+        all_sids = [int(r[0]) for r in self.session.execute(select(t_st.c.station_id)).all()]
+        absent = [sid for sid in all_sids if sid not in present_station_ids]
+        if not absent:
+            return (0, 0, 0)
+        
+        # Markets: delete baseline rows (from_live=0) with modified <= json_ts
+        del_m = self.session.execute(
+            t_si.delete().where(
+                and_(
+                    t_si.c.station_id.in_(absent),
+                    t_si.c.from_live == 0,
+                    or_(t_si.c.modified.is_(None), t_si.c.modified <= json_ts),
+                )
+            )
+        ).rowcount or 0
+        
+        # Vendors: delete rows with modified <= json_ts
+        del_u = self.session.execute(
+            tables["UpgradeVendor"].delete().where(
+                and_(t_uv.c.station_id.in_(absent), or_(t_uv.c.modified.is_(None), t_uv.c.modified <= json_ts))
+            )
+        ).rowcount or 0
+        del_s = self.session.execute(
+            tables["ShipVendor"].delete().where(
+                and_(t_sv.c.station_id.in_(absent), or_(t_sv.c.modified.is_(None), t_sv.c.modified <= json_ts))
+            )
+        ).rowcount or 0
+        
+        return (int(del_m), int(del_u), int(del_s))
+    
+    def _sync_market_block_fast(
+        self,
+        tables: dict[str, Table],
+        categories: dict[str, int],
+        *,
+        station_id: int,
+        commodities: list[dict[str, Any]],
+        ts_sp: datetime,
+    ) -> tuple[int, int]:
+        """
+        Fast, set-based market sync for one station.
+        
+        Returns: (number_of_inserts_or_updates_on_StationItem, deletions_count).
+        """
+        t_item, t_si = tables["Item"], tables["StationItem"]
+        
+        item_rows: list[dict[str, Any]] = []
+        link_rows: list[dict[str, Any]] = []
+        keep_ids: set[int] = set()
+        
+        for co in commodities:
+            if not isinstance(co, dict):
+                continue
+            fdev_id = co.get("commodityId")
+            name = co.get("name")
+            cat_name = co.get("category")
+            if fdev_id is None or name is None or cat_name is None:
+                continue
+            
+            cat_id = categories.get(str(cat_name).lower())
+            if cat_id is None:
+                raise CleanExit(f'Unknown commodity category "{cat_name}"')
+            
+            keep_ids.add(int(fdev_id))
+            item_rows.append({
+                "item_id": fdev_id,
+                "name": name,
+                "category_id": cat_id,
+                "fdev_id": fdev_id,
+                "ui_order": 0,
+            })
+            
+            demand = co.get("demand")
+            supply = co.get("supply")
+            buy = co.get("buyPrice")
+            sell = co.get("sellPrice")
+            
+            link_rows.append({
+                "station_id": station_id,
+                "item_id": fdev_id,
+                "demand_price": sell,
+                "demand_units": demand,
+                "demand_level": -1,
+                "supply_price": buy,
+                "supply_units": supply,
+                "supply_level": -1,
+                "from_live": 0,
+                "modified": ts_sp,
+            })
+        
+        # 1) Upsert Items (simple)
+        if item_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_simple(
+                    self.session, t_item, rows=item_rows,
+                    key_cols=("item_id",),
+                    update_cols=("name", "category_id", "fdev_id", "ui_order"),
+                )
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_simple(
+                    self.session, t_item, rows=item_rows,
+                    key_cols=("item_id",),
+                    update_cols=("name", "category_id", "fdev_id", "ui_order"),
+                )
+            else:
+                for r in item_rows:
+                    exists = self.session.execute(
+                        select(t_item.c.item_id).where(t_item.c.item_id == r["item_id"])
+                    ).first()
+                    if exists is None:
+                        self.session.execute(insert(t_item).values(**r))
+                    else:
+                        self.session.execute(
+                            update(t_item).where(t_item.c.item_id == r["item_id"]).values(
+                                name=r["name"], category_id=r["category_id"], fdev_id=r["fdev_id"], ui_order=r["ui_order"]
+                            )
+                        )
+        
+        # 2) Compute effective inserts/updates for StationItem (pre-check modified), then upsert
+        wrote = 0
+        if link_rows:
+            existing = {
+                (int(r[0]), int(r[1])): (r[2] or None)
+                for r in self.session.execute(
+                    select(t_si.c.station_id, t_si.c.item_id, t_si.c.modified).where(
+                        and_(t_si.c.station_id == station_id, t_si.c.item_id.in_(keep_ids))
+                    )
+                ).all()
+            }
+            to_insert = {
+                (station_id, rid) for rid in keep_ids
+                if (station_id, rid) not in existing
+            }
+            to_update = {
+                (station_id, rid) for rid, mod in ((rid, existing.get((station_id, rid))) for rid in keep_ids)
+                if (mod is None) or (ts_sp is not None and ts_sp > mod)
+            }
+            wrote = len(to_insert) + len(to_update)
+            
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_modified(
+                    self.session, t_si, rows=link_rows,
+                    key_cols=("station_id", "item_id"),
+                    modified_col="modified",
+                    update_cols=("demand_price", "demand_units", "demand_level",
+                                 "supply_price", "supply_units", "supply_level", "from_live"),
+                )
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_modified(
+                    self.session, t_si, rows=link_rows,
+                    key_cols=("station_id", "item_id"),
+                    modified_col="modified",
+                    update_cols=("demand_price", "demand_units", "demand_level",
+                                 "supply_price", "supply_units", "supply_level", "from_live"),
+                )
+            else:
+                for r in link_rows:
+                    row = self.session.execute(
+                        select(t_si.c.modified).where(and_(
+                            t_si.c.station_id == r["station_id"],
+                            t_si.c.item_id == r["item_id"],
+                        ))
+                    ).first()
+                    if row is None:
+                        self.session.execute(insert(t_si).values(**r))
+                    else:
+                        dbm = row[0]
+                        if dbm is None or r["modified"] > dbm:
+                            self.session.execute(
+                                update(t_si)
+                                .where(and_(t_si.c.station_id == r["station_id"], t_si.c.item_id == r["item_id"]))
+                                .values(**r)
+                            )
+        
+        # 3) Delete baseline rows missing from JSON, not newer than ts_sp
+        delc = 0
+        base_where = and_(
+            t_si.c.station_id == station_id,
+            t_si.c.from_live == 0,
+            or_(t_si.c.modified.is_(None), t_si.c.modified <= ts_sp),
+        )
+        if keep_ids:
+            delete_stmt = t_si.delete().where(and_(base_where, ~t_si.c.item_id.in_(keep_ids)))
+        else:
+            delete_stmt = t_si.delete().where(base_where)
+        
+        res = self.session.execute(delete_stmt)
+        try:
+            delc = int(res.rowcount or 0)
+        except Exception:
+            delc = 0
+        
+        return wrote, delc
+    
+    # ------------------------------
+    # Lifecycle hooks
+    #
+    def run(self) -> bool:
+        """
+        Full orchestrator: acquisition → bootstrap → EDCD preload → import → rares → export.
+        Returns False to keep default flow suppressed.
+        """
+        started = time.time()
+        
+        if self.getOption("pricesonly"):
+            try:
+                self._print("Regenerating TradeDangerous.prices …")
+                cache.regeneratePricesFile(self.tdb, self.tdenv)
+                self._print("Prices file generated.")
+            except Exception as e:
+                self._error(f"Prices regeneration failed: {e!r}")
+                return False
+            return False
+        
+        # Acquire Spansh JSON
+        try:
+            source_path = self._acquire_source()
+        except CleanExit as ce:
+            self._warn(str(ce))
+            return False
+        except Exception as e:
+            self._error(f"Acquisition failed: {e!r}")
+            return False
+        
+        # -------- Bootstrap DB (no cache rebuild here) --------
+        try:
+            backend  = self.tdb.engine.dialect.name.lower()
+            data_dir = Path(getattr(self.tdenv, "dataDir", getattr(self.tdb, "dataDir", "data")))
+            metadata = getattr(self.tdb, "metadata", None)
+            
+            summary = ensure_fresh_db(
+                backend=backend,
+                engine=self.tdb.engine,
+                data_dir=data_dir,
+                metadata=metadata,
+                mode="auto",
+                tdb=self.tdb,
+                tdenv=self.tdenv,
+                rebuild=False,   # do not run buildCache here
+            )
+            self._print(
+                f"DB bootstrap: action={summary.get('action','kept')} "
+                f"reason={summary.get('reason','ok')} backend={summary.get('backend')}"
+            )
+            
+            # No valid DB? Create full schema now (SQLite from canonical SQL; MariaDB via ORM)
+            if summary.get("action") == "needs_rebuild":
+                db_path = Path(self.tdb.engine.url.database or (data_dir / "TradeDangerous.db"))  # SQLite only
+                self._print("No valid DB detected — creating full schema…")
+                reset_db(self.tdb.engine, db_path=db_path)
+                
+                # Seed 'Added' once on a fresh schema
+                self.session = self._open_session()
+                self._seed_added_from_templates(self.session)
+                self.session.commit()
+                self._safe_close_session()
+        
+        except Exception as e:
+            self._error(f"Database bootstrap failed: {e!r}")
+            return False
+        
+        # -------- Session + batch + reflection --------
+        try:
+            self.session = self._open_session()
+            self.batch_size = self._resolve_batch_size()
+            tables = self._reflect_tables(self.session.get_bind())
+        except Exception as e:
+            self._error(f"Failed to open/reflect DB session: {e!r}")
+            return False
+        
+        # -------- EDCD preloads (hardcoded URLs; can be disabled) --------
+        edcd = self._acquire_edcd_files()
+        
+        # Categories (add-only) — COMMIT immediately so they persist even if later phases fail.
+        try:
+            if edcd.get("commodity"):
+                added = self._edcd_import_categories_add_only(self.session, tables, edcd["commodity"])
+                if added:
+                    self._print(f"EDCD categories: added {added} new categories")
+                self.session.commit()
+        except CleanExit as ce:
+            self._warn(str(ce))
+            return False
+        except Exception as e:
+            self._warn(f"EDCD categories skipped due to error: {e!r}")
+        
+        # FDev catalogs (outfitting, shipyard) — COMMIT immediately as well.
+        try:
+            if edcd.get("outfitting") and edcd.get("shipyard"):
+                u, s = self._edcd_import_fdev_catalogs(
+                    self.session, tables,
+                    outfitting_csv=edcd["outfitting"],
+                    shipyard_csv=edcd["shipyard"],
+                )
+                if (u + s) > 0:
+                    self._print(f"EDCD FDev: Outfitting upserts={u:,}  Shipyard upserts={s:,}")
+                self.session.commit()
+        except Exception as e:
+            self._warn(f"EDCD FDev catalogs skipped due to error: {e!r}")
+        
+        # Load categories (may have grown) before Spansh import
+        try:
+            categories = self._load_categories(self.session, tables)
+        except Exception as e:
+            self._error(f"Failed to load categories: {e!r}")
+            return False
+        
+        # -------- Import Spansh JSON --------
+        try:
+            if self._debug_level < 1:
+                self._print("This will take at least several minutes.")
+                self._print("You can increase verbosity (-v) to get a sense of progress")
+            self._print("Importing spansh data")
+            stats = self._import_stream(source_path, categories, tables)
+            self._end_live_status()
+            
+            mk_e = stats.get("market_writes", 0) + stats.get("market_stations", 0)
+            of_e = stats.get("outfit_writes", 0) + stats.get("outfit_stations", 0)
+            sh_e = stats.get("ship_writes", 0) + stats.get("ship_stations", 0)
+            self._print(
+                f"Import complete — systems: {stats.get('systems',0):,}  "
+                f"stations: {stats.get('stations',0):,}  "
+                f"evaluated: markets≈{mk_e:,} outfitters≈{of_e:,} shipyards≈{sh_e:,}  "
+                f"kept: markets≈{stats.get('market_stations',0):,} outfitters≈{stats.get('outfit_stations',0):,} shipyards≈{stats.get('ship_stations',0):,}"
+            )
+        except CleanExit as ce:
+            self._warn(str(ce))
+            self._safe_close_session()
+            return False
+        except Exception as e:
+            self._error(f"Import failed: {e!r}")
+            self._safe_close_session()
+            return False
+        
+        # Enforce Item.ui_order
+        try:
+            t0 = time.time()
+            self._enforce_ui_order(self.session, tables)
+            self._print(f"ui_order enforced in {time.time()-t0:.2f}s")
+        except Exception as e:
+            self._error(f"ui_order enforcement failed: {e!r}")
+            self._safe_close_session()
+            return False
+        
+        # Final commit for import phase
+        try:
+            self.session.commit()
+        except Exception as e:
+            self._warn(f"Commit failed at end of import; rolling back. Cause: {e!r}")
+            self.session.rollback()
+            self._safe_close_session()
+            return False
+        
+        self._safe_close_session()
+        
+        # -------- Rares (prefer EDCD; fallback to template) --------
+        try:
+            t0 = time.time()
+            if edcd.get("rares"):
+                self._import_rareitems_edcd(edcd["rares"])
+            else:
+                self._import_rareitems()
+            self._print(f"Rares imported in {time.time()-t0:.2f}s")
+        except CleanExit as ce:
+            self._warn(str(ce))
+            return False
+        except Exception as e:
+            self._error(f"RareItem import failed: {e!r}")
+            return False
+        
+        # -------- Export (uses your parallel exporter already present) --------
+        try:
+            self._export_and_mirror()  # timing + final print handled inside
+        except Exception as e:
+            self._error(f"Export failed: {e!r}")
+            return False
+        
+        elapsed = self._format_hms(time.time() - started)
+        self._print(f"{elapsed}  Done")
+        return False
+
+
+    
+    def finish(self) -> bool:
+        """No-op: handled in run(); finish() won’t be called."""
+        return True
+    
+    # ------------------------------
+    # Acquisition (url/file/stdin)
+    # ------------------------------
+    
+    def _acquire_source(self) -> Path:
+        """Return a readable filesystem path to the JSON source (tmp/)."""
+        url = self.getOption("url")
+        file_ = self.getOption("file")
+        cache_path = self.tmp_dir / "galaxy_stations.json"
+        
+        if file_:
+            if file_ == "-":
+                self._print("Reading Spansh dump from stdin …")
+                self._write_stream_to_file(sys.stdin.buffer, cache_path)
+                return cache_path
+            src = Path(file_)
+            if not src.exists() or not src.is_file():
+                raise CleanExit(f"Local file not found: {src}")
+            return src.resolve()
+        
+        if not url:
+            url = DEFAULT_URL
+        
+        # Pass a friendly label so progress says “Spansh dump”
+        return self._download_with_cache(url, cache_path, label="Spansh dump")
+    
+    def _download_with_cache(self, url: str, cache_path: Path, *, label: str = "download") -> Path:
+        """Conditional download with HEAD Last-Modified and atomic .part."""
+        remote_lm: Optional[datetime] = None
+        try:
+            req = urllib.request.Request(url, method="HEAD")
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                lm_header = resp.headers.get("Last-Modified")
+                if lm_header:
+                    try:
+                        remote_lm = parsedate_to_datetime(lm_header).astimezone(timezone.utc).replace(tzinfo=None)
+                    except Exception:
+                        remote_lm = None
+        except Exception:
+            pass
+        
+        if cache_path.exists() and remote_lm:
+            local_mtime = datetime.fromtimestamp(cache_path.stat().st_mtime, tz=timezone.utc).replace(tzinfo=None)
+            if local_mtime >= remote_lm:
+                self._print(f"Remote not newer; using cached {label}")
+                return cache_path
+        
+        self._print(f"Downloading {label} from {url} …")
+        part = cache_path.with_suffix(cache_path.suffix + ".part")
+        if part.exists():
+            try:
+                part.unlink()
+            except Exception:
+                pass
+        
+        req = urllib.request.Request(url, method="GET")
+        connect_timeout = 30
+        chunk = 8 * 1024 * 1024  # 8 MiB
+        
+        try:
+            with urllib.request.urlopen(req, timeout=connect_timeout) as resp, open(part, "wb") as fh:
+                total_hdr = resp.headers.get("Content-Length")
+                total = int(total_hdr) if total_hdr and total_hdr.isdigit() else None
+                downloaded = 0
+                start = time.time()
+                
+                while True:
+                    data = resp.read(chunk)
+                    if not data:
+                        break
+                    fh.write(data)
+                    downloaded += len(data)
+                    self._download_progress(downloaded, total, start, label=label)
+            
+            part.replace(cache_path)
+            
+            # Set mtime to Last-Modified if present on GET
+            lm_header = None
+            try:
+                with urllib.request.urlopen(urllib.request.Request(url, method="HEAD"), timeout=10) as head2:
+                    lm_header = head2.headers.get("Last-Modified")
+            except Exception:
+                pass
+            if lm_header:
+                try:
+                    got_lm = parsedate_to_datetime(lm_header).astimezone(timezone.utc).replace(tzinfo=None)
+                    ts = got_lm.replace(tzinfo=timezone.utc).timestamp()
+                    os.utime(cache_path, (ts, ts))
+                except Exception:
+                    pass
+        
+        except Exception as e:
+            try:
+                if part.exists():
+                    part.unlink()
+            except Exception:
+                pass
+            raise CleanExit(f"Download failed or timed out for {label}; skipping run ({e!r})") from None
+        
+        self._print(f'Download complete: {label} → "{cache_path}"')
+        return cache_path
+    
+    def _download_progress(self, downloaded: int, total: Optional[int], start_ts: float, *, label: str = "download") -> None:
+        now = time.time()
+        if now - self._last_progress_time < 0.5 and self._debug_level < 1:
+            return
+        self._last_progress_time = now
+        
+        rate = downloaded / max(now - start_ts, 1e-9)
+        if total:
+            pct = (downloaded / total) * 100.0
+            msg = f"{label}: {self._fmt_bytes(downloaded)} / {self._fmt_bytes(total)} ({pct:5.1f}%)  {self._fmt_bytes(rate)}/s"
+        else:
+            msg = f"{label}: {self._fmt_bytes(downloaded)} read  {self._fmt_bytes(rate)}/s"
+        self._live_status(msg)
+    
+    def _write_stream_to_file(self, stream: io.BufferedReader, dest: Path) -> None:
+        part = dest.with_suffix(dest.suffix + ".part")
+        if part.exists():
+            try:
+                part.unlink()
+            except Exception:
+                pass
+        written = 0
+        start = time.time()
+        try:
+            with open(part, "wb") as fh:
+                while True:
+                    buf = stream.read(8 * 1024 * 1024)
+                    if not buf:
+                        break
+                    fh.write(buf)
+                    written += len(buf)
+                    self._download_progress(written, None, start)
+            part.replace(dest)
+        except Exception as e:
+            try:
+                if part.exists():
+                    part.unlink()
+            except Exception:
+                pass
+            raise CleanExit(f"Failed to read stdin into tmp file: {e!r})") from None
+    
+    # ------------------------------
+    # DB session / reflection
+    # ------------------------------
+    def _open_session(self) -> Session:
+        """
+        Create a DB session and apply per-connection bulk settings.
+        """
+        if hasattr(self.tdb, "Session") and callable(self.tdb.Session):
+            sess = self.tdb.Session()
+        elif hasattr(db_utils, "get_session"):
+            sess = db_utils.get_session(self.tdb.engine)
+        else:
+            raise RuntimeError("No Session factory available")
+        
+        # SQLite pragmas (non-fatal)
+        try:
+            if db_utils.is_sqlite(sess):
+                db_utils.sqlite_set_bulk_pragmas(sess)
+        except Exception:
+            pass
+        
+        # MySQL/MariaDB session tuning (non-fatal)
+        try:
+            if db_utils.is_mysql(sess):
+                db_utils.mysql_set_bulk_session(sess)
+        except Exception:
+            pass
+        
+        return sess
+    
+    def _reflect_tables(self, engine: Engine) -> dict[str, Table]:
+        meta = MetaData()
+        names = [
+            "System", "Station", "Item", "Category", "StationItem",
+            "Ship", "ShipVendor", "Upgrade", "UpgradeVendor",
+            "FDevOutfitting", "FDevShipyard", "RareItem",
+        ]
+        return {n: Table(n, meta, autoload_with=engine) for n in names}
+    
+    # ------------------------------
+    # Import (streaming JSON → upserts)
+    # ------------------------------
+    def _import_stream(self, source_path: Path, categories: dict[str, int], tables: dict[str, Table]) -> dict[str, int]:
+        """
+        Streaming importer with service-level maxage gating (FK-safe), using per-row rules.
+        
+        FIXES:
+        - Batch commits now honor utils.get_import_batch_size() across *all* parent/child ops.
+        - System/Station increments are counted in stats and batch_ops.
+        - Commit checks occur before each station is processed (outside advisory lock scope),
+          reducing long transactions and making Ctrl-C loss less likely.
+        """
+        batch_ops = 0
+        stats = {
+            "systems": 0, "stations": 0,
+            "market_stations": 0, "outfit_stations": 0, "ship_stations": 0,
+            "market_writes": 0, "outfit_writes": 0, "ship_writes": 0,
+            "commodities": 0,
+        }
+        
+        # NEW: initialize parse metrics for _progress_line(); iterator keeps these updated
+        self._parse_bytes = 0
+        self._parse_rate = 0.0
+        
+        maxage_days = float(self.getOption("maxage")) if self.getOption("maxage") else None
+        maxage_td = timedelta(days=maxage_days) if maxage_days is not None else None
+        now_utc = datetime.utcnow()
+        
+        try:
+            json_ts = datetime.fromtimestamp(os.path.getmtime(source_path), tz=timezone.utc).replace(tzinfo=None)
+        except Exception:
+            json_ts = datetime.utcfromtimestamp(0)
+        
+        seen_station_ids: set[int] = set()
+        force_baseline = bool(self.getOption("force_baseline"))
+        
+        def recent(ts: Optional[datetime]) -> bool:
+            if ts is None:
+                return not maxage_td  # None -> True, otherwise False.
+            if maxage_td is None:
+                return True
+            return (now_utc - ts) <= maxage_td
+        
+        def svc_ts(st: dict[str, Any], key: str) -> Optional[datetime]:
+            obj = st.get(key) or {}
+            if not isinstance(obj, dict):
+                return None
+            return self._parse_ts(obj.get("updateTime"))
+        
+        with open(source_path, "rb") as fh:
+            for sys_idx, system_obj in enumerate(self._iter_top_level_json_array(fh), 1):
+                sys_id64 = system_obj.get("id64")
+                sys_name = system_obj.get("name")
+                coords = system_obj.get("coords") or {}
+                if sys_id64 is None or sys_name is None or not isinstance(coords, dict):
+                    if self._debug_level >= 3:
+                        self._warn(f"Skipping malformed system object at index {sys_idx}")
+                    continue
+                
+                self._trace(phase="system", decision="consider", name=sys_name, id64=sys_id64)
+                
+                # Collect stations (top-level + body-embedded)
+                stations: list[dict[str, Any]] = []
+                if isinstance(system_obj.get("stations"), list):
+                    stations.extend(system_obj["stations"])
+                bodies = system_obj.get("bodies") or []
+                if isinstance(bodies, list):
+                    for b in bodies:
+                        if isinstance(b, dict):
+                            stl = b.get("stations")
+                            if isinstance(stl, list):
+                                stations.extend(stl)
+                
+                # --- System upsert ---
+                t_system = tables["System"]
+                x, y, z = coords.get("x"), coords.get("y"), coords.get("z")
+                sys_modified = self._parse_ts(system_obj.get("updateTime"))
+                self._upsert_system(t_system, int(sys_id64), str(sys_name), x, y, z, sys_modified)
+                
+                # Count system progress and participate in batching
+                stats["systems"] += 1
+                batch_ops += 1
+                
+                imported_station_modifieds: list[datetime] = []
+                
+                for st in stations:
+                    # Periodic commit BEFORE processing the next station (outside any advisory locks)
+                    if (self.batch_size is not None) and (batch_ops >= self.batch_size):
+                        try:
+                            self.session.commit()
+                            batch_ops = 0
+                        except Exception as e:
+                            self._warn(f"Batch commit failed; rolling back. Cause: {e!r}")
+                            self.session.rollback()
+                    
+                    name = st.get("name")
+                    sid = st.get("id")
+                    if not isinstance(name, str) or sid is None:
+                        continue
+                    station_id = int(sid)
+                    seen_station_ids.add(station_id)
+                    stats["stations"] += 1
+                    # Count at least one op per station so batching still progresses even if no vendor writes occur
+                    batch_ops += 1
+                    
+                    # NEW: drive live progress from here (throttled inside _progress_line)
+                    self._progress_line(stats)
+                    
+                    # Flags/timestamps
+                    has_market = bool(st.get("hasMarket") or ("market" in st))
+                    has_outfit = bool(st.get("hasOutfitting") or ("outfitting" in st))
+                    has_ship   = bool(st.get("hasShipyard") or ("shipyard" in st))
+                    mkt_ts  = svc_ts(st, "market")
+                    outf_ts = svc_ts(st, "outfitting")
+                    ship_ts = svc_ts(st, "shipyard")
+                    mkt_fresh  = recent(mkt_ts)
+                    outf_fresh = recent(outf_ts)
+                    ship_fresh = recent(ship_ts)
+                    
+                    # Station upsert (idempotent)
+                    t_station = tables["Station"]
+                    type_id, planetary = self._map_station_type(st.get("type"))
+                    pads = st.get("landingPads") or {}
+                    max_pad = self._derive_pad_size(pads)
+                    sflags = {
+                        "market": "Y" if has_market else "N",
+                        "blackmarket": "?" if st.get("hasBlackmarket") is None else ("Y" if st.get("hasBlackmarket") else "N"),
+                        "shipyard": "Y" if has_ship else "N",
+                        "outfitting": "Y" if has_outfit else "N",
+                        "rearm": "?" if st.get("hasRearm") is None else ("Y" if st.get("hasRearm") else "N"),
+                        "refuel": "?" if st.get("hasRefuel") is None else ("Y" if st.get("hasRefuel") else "N"),
+                        "repair": "?" if st.get("hasRepair") is None else ("Y" if st.get("hasRepair") else "N"),
+                    }
+                    st_modified = self._parse_ts(st.get("updateTime"))
+                    if st_modified:
+                        imported_station_modifieds.append(st_modified)
+                    
+                    ls_from_star_val = st.get("distanceToArrival", 0)
+                    try:
+                        if ls_from_star_val is None:
+                            ls_from_star_val = 0
+                        else:
+                            ls_from_star_val = max(int(float(ls_from_star_val)), 0)
+                    except Exception:
+                        ls_from_star_val = 0
+                    
+                    self._upsert_station(
+                        t_station, station_id=int(station_id), system_id=int(sys_id64), name=name,
+                        ls_from_star=ls_from_star_val, max_pad=max_pad,
+                        type_id=int(type_id), planetary=planetary, sflags=sflags, modified=st_modified
+                    )
+                    
+                    # ----------------------------
+                    # Ship vendor
+                    # ----------------------------
+                    if has_ship and ship_fresh:
+                        ships = (st.get("shipyard") or {}).get("ships") or []
+                        if isinstance(ships, list) and ships:
+                            if force_baseline:
+                                wrote, _, delc = self._apply_vendor_block_per_rules(
+                                    tables["ShipVendor"], station_id, (s.get("shipId") for s in ships if isinstance(s, dict)),
+                                    ship_ts, id_col="ship_id",
+                                )
+                                if wrote or delc:
+                                    stats["ship_writes"] += 1
+                                    batch_ops += (wrote + delc)
+                                stats["ship_stations"] += 1
+                            else:
+                                wrote, delc = self._sync_vendor_block_fast(
+                                    tables, station_id=station_id, entries=ships, ts_sp=ship_ts, kind="ship"
+                                )
+                                if wrote or delc:
+                                    stats["ship_writes"] += 1
+                                    batch_ops += (wrote + delc)
+                                stats["ship_stations"] += 1
+                        else:
+                            stats["ship_stations"] += 1
+                    
+                    # ----------------------------
+                    # Outfitting vendor
+                    # ----------------------------
+                    if has_outfit and outf_fresh:
+                        modules = (st.get("outfitting") or {}).get("modules") or []
+                        if isinstance(modules, list) and modules:
+                            if force_baseline:
+                                wrote = self._upsert_outfitting(tables, station_id, modules, outf_ts)
+                                _, _, delc = self._apply_vendor_block_per_rules(
+                                    tables["UpgradeVendor"], station_id,
+                                    (m.get("moduleId") for m in modules if isinstance(m, dict)),
+                                    outf_ts, id_col="upgrade_id",
+                                )
+                                if wrote or delc:
+                                    stats["outfit_writes"] += 1
+                                    batch_ops += (wrote + delc)
+                                stats["outfit_stations"] += 1
+                            else:
+                                wrote, delc = self._sync_vendor_block_fast(
+                                    tables, station_id=station_id, entries=modules, ts_sp=outf_ts, kind="module"
+                                )
+                                if wrote or delc:
+                                    stats["outfit_writes"] += 1
+                                    batch_ops += (wrote + delc)
+                                stats["outfit_stations"] += 1
+                        else:
+                            stats["outfit_stations"] += 1
+                    
+                    # ----------------------------
+                    # Market (commit check already happened before this station)
+                    # ----------------------------
+                    if has_market and mkt_fresh:
+                        commodities = (st.get("market") or {}).get("commodities") or []
+                        if isinstance(commodities, list) and commodities:
+                            # The advisory lock context pins lock + DML to the same connection/txn.
+                            with station_advisory_lock(self.session, station_id, timeout_seconds=0.2, max_retries=4) as got:
+                                if not got:
+                                    # Could not acquire; try this station on a later pass
+                                    continue
+                                
+                                self._trace(phase="market", decision="process",
+                                            station_id=station_id, commodities=len(commodities))
+                                
+                                if force_baseline:
+                                    wrote_i, wrote_si = self._upsert_market(
+                                        tables, categories, station_id, commodities, mkt_ts
+                                    )
+                                    # Remove any extras unconditionally (baseline reset)
+                                    t_si = tables["StationItem"]
+                                    keep_ids = {
+                                        int(co.get("commodityId"))
+                                        for co in commodities
+                                        if isinstance(co, dict) and co.get("commodityId") is not None
+                                    }
+                                    if keep_ids:
+                                        self.session.execute(
+                                            t_si.delete().where(
+                                                and_(t_si.c.station_id == station_id, ~t_si.c.item_id.in_(keep_ids))
+                                            )
+                                        )
+                                    stats["commodities"] += wrote_si
+                                    if wrote_si or wrote_i:
+                                        stats["market_writes"] += 1
+                                        batch_ops += (wrote_i + wrote_si)
+                                    stats["market_stations"] += 1
+                                else:
+                                    wrote_links, delc = self._sync_market_block_fast(
+                                        tables, categories,
+                                        station_id=station_id,
+                                        commodities=commodities,
+                                        ts_sp=mkt_ts,
+                                    )
+                                    if wrote_links or delc:
+                                        stats["market_writes"] += 1
+                                        batch_ops += (wrote_links + delc)
+                                    stats["market_stations"] += 1
+                        else:
+                            stats["market_stations"] += 1
+        
+        # Baseline absent-station cleanup (global, after full stream)
+        # We only remove baseline content (from_live=0 for markets; vendor links)
+        # and only where modified <= json_ts, so anything newer (e.g. live/ZMQ) is preserved.
+        try:
+            if force_baseline and seen_station_ids:
+                m_del, u_del, s_del = self._cleanup_absent_stations(
+                    tables,
+                    present_station_ids=seen_station_ids,
+                    json_ts=json_ts,
+                )
+                if (m_del + u_del + s_del) > 0 and self._debug_level >= 1:
+                    self._print(
+                        f"Baseline cleanup: markets={m_del:,}  upgrades={u_del:,}  ships={s_del:,}"
+                    )
+        except Exception as e:
+            self._warn(f"Absent-station cleanup skipped due to error: {e!r}")
+        
+        return stats
+
+
+    # ------------------------------
+    # Upsert helpers
+    # ------------------------------
+
+
+    def _upsert_system(
+        self, t_system: Table, system_id: int, name: str,
+        x: Optional[float], y: Optional[float], z: Optional[float],
+        modified: Optional[datetime],
+    ) -> None:
+        """
+        Upsert System with timestamp guard.
+        'added' policy (when column exists):
+          - INSERT: set added=20 (EDSM).
+          - UPDATE: do not overwrite, unless existing added IS NULL → set to 20.
+        """
+        if modified is None:
+            modified = datetime.utcfromtimestamp(0)
+        
+        has_added_col = hasattr(t_system.c, "added")
+        
+        row = {
+            "system_id": system_id,
+            "name": name,
+            "pos_x": x, "pos_y": y, "pos_z": z,
+            "modified": modified,
+        }
+        if has_added_col:
+            row["added"] = 20  # EDSM on INSERT
+        
+        if db_utils.is_sqlite(self.session):
+            db_utils.sqlite_upsert_modified(
+                self.session, t_system,
+                rows=[row],
+                key_cols=("system_id",),
+                modified_col="modified",
+                update_cols=("name", "pos_x", "pos_y", "pos_z"),
+            )
+            if has_added_col:
+                self.session.execute(
+                    update(t_system)
+                    .where((t_system.c.system_id == system_id) & (t_system.c.added.is_(None)))
+                    .values(added=20)
+                )
+            return
+        
+        if db_utils.is_mysql(self.session):
+            db_utils.mysql_upsert_modified(
+                self.session, t_system,
+                rows=[row],
+                key_cols=("system_id",),
+                modified_col="modified",
+                update_cols=("name", "pos_x", "pos_y", "pos_z"),
+            )
+            if has_added_col:
+                self.session.execute(
+                    update(t_system)
+                    .where((t_system.c.system_id == system_id) & (t_system.c.added.is_(None)))
+                    .values(added=20)
+                )
+            return
+        
+        # Generic fallback
+        sel_cols = [t_system.c.modified]
+        if has_added_col:
+            sel_cols.append(t_system.c.added)
+        existing = self.session.execute(
+            select(*sel_cols).where(t_system.c.system_id == system_id)
+        ).first()
+        
+        if existing is None:
+            self.session.execute(insert(t_system).values(**row))
+        else:
+            db_modified = existing[0]
+            values = {"name": name, "pos_x": x, "pos_y": y, "pos_z": z}
+            if db_modified is None or modified > db_modified:
+                values["modified"] = modified
+            self.session.execute(
+                update(t_system)
+                .where(t_system.c.system_id == system_id)
+                .values(**values)
+            )
+            if has_added_col:
+                db_added = existing[1] if len(existing) > 1 else None
+                if db_added is None:
+                    self.session.execute(
+                        update(t_system)
+                        .where((t_system.c.system_id == system_id) & (t_system.c.added.is_(None)))
+                        .values(added=20)
+                    )
+    
+    def _upsert_station(
+        self, t_station: Table, station_id: int, system_id: int, name: str,
+        ls_from_star: Optional[float], max_pad: str, type_id: int, planetary: str,
+        sflags: dict[str, str], modified: Optional[datetime],
+    ) -> None:
+        """
+        Upsert Station with timestamp guard.
+        """
+        if modified is None:
+            modified = datetime.utcfromtimestamp(0)
+        
+        if db_utils.is_sqlite(self.session):
+            db_utils.sqlite_upsert_modified(
+                self.session, t_station,
+                rows=[{
+                    "station_id": station_id,
+                    "system_id": system_id,
+                    "name": name,
+                    "ls_from_star": ls_from_star,
+                    "max_pad_size": max_pad,
+                    "type_id": type_id,
+                    "planetary": planetary,
+                    "market": sflags["market"],
+                    "blackmarket": sflags["blackmarket"],
+                    "shipyard": sflags["shipyard"],
+                    "outfitting": sflags["outfitting"],
+                    "rearm": sflags["rearm"],
+                    "refuel": sflags["refuel"],
+                    "repair": sflags["repair"],
+                    "modified": modified,
+                }],
+                key_cols=("station_id",),
+                modified_col="modified",
+                update_cols=(
+                    "system_id", "name", "ls_from_star", "max_pad_size", "type_id", "planetary",
+                    "market", "blackmarket", "shipyard", "outfitting", "rearm", "refuel", "repair",
+                ),
+            )
+            return
+        
+        if db_utils.is_mysql(self.session):
+            db_utils.mysql_upsert_modified(
+                self.session, t_station,
+                rows=[{
+                    "station_id": station_id,
+                    "system_id": system_id,
+                    "name": name,
+                    "ls_from_star": ls_from_star,
+                    "max_pad_size": max_pad,
+                    "type_id": type_id,
+                    "planetary": planetary,
+                    "market": sflags["market"],
+                    "blackmarket": sflags["blackmarket"],
+                    "shipyard": sflags["shipyard"],
+                    "outfitting": sflags["outfitting"],
+                    "rearm": sflags["rearm"],
+                    "refuel": sflags["refuel"],
+                    "repair": sflags["repair"],
+                    "modified": modified,
+                }],
+                key_cols=("station_id",),
+                modified_col="modified",
+                update_cols=(
+                    "system_id", "name", "ls_from_star", "max_pad_size", "type_id", "planetary",
+                    "market", "blackmarket", "shipyard", "outfitting", "rearm", "refuel", "repair",
+                ),
+            )
+            return
+        
+        # Generic fallback
+        row = self.session.execute(
+            select(t_station.c.system_id, t_station.c.modified)
+            .where(t_station.c.station_id == station_id)
+        ).first()
+        
+        if row is None:
+            self.session.execute(
+                insert(t_station).values(
+                    station_id=station_id,
+                    system_id=system_id,
+                    name=name,
+                    ls_from_star=ls_from_star,
+                    max_pad_size=max_pad,
+                    type_id=type_id,
+                    planetary=planetary,
+                    market=sflags["market"],
+                    blackmarket=sflags["blackmarket"],
+                    shipyard=sflags["shipyard"],
+                    outfitting=sflags["outfitting"],
+                    rearm=sflags["rearm"],
+                    refuel=sflags["refuel"],
+                    repair=sflags["repair"],
+                    modified=modified,
+                )
+            )
+        else:
+            db_system_id, db_modified = row
+            values = {
+                "name": name,
+                "ls_from_star": ls_from_star,
+                "max_pad_size": max_pad,
+                "type_id": type_id,
+                "planetary": planetary,
+                "market": sflags["market"],
+                "blackmarket": sflags["blackmarket"],
+                "shipyard": sflags["shipyard"],
+                "outfitting": sflags["outfitting"],
+                "rearm": sflags["rearm"],
+                "refuel": sflags["refuel"],
+                "repair": sflags["repair"],
+            }
+            if db_system_id != system_id:
+                values["system_id"] = system_id
+            if db_modified is None or modified > db_modified:
+                values["modified"] = modified
+            
+            self.session.execute(
+                update(t_station)
+                .where(t_station.c.station_id == station_id)
+                .values(**values)
+            )
+    
+    def _upsert_shipyard(self, tables: dict[str, Table], station_id: int, ships: list[dict[str, Any]], ts: datetime) -> int:
+        t_ship, t_vendor = tables["Ship"], tables["ShipVendor"]
+        ship_rows, vendor_rows = [], []
+        
+        for sh in ships:
+            ship_id = sh.get("shipId")
+            name = sh.get("name")
+            if ship_id is None or name is None:
+                continue
+            ship_rows.append({"ship_id": ship_id, "name": name})
+            vendor_rows.append({"ship_id": ship_id, "station_id": station_id, "modified": ts})
+        
+        if ship_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_simple(self.session, t_ship, rows=ship_rows, key_cols=("ship_id",), update_cols=("name",))
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_simple(self.session, t_ship, rows=ship_rows, key_cols=("ship_id",), update_cols=("name",))
+            else:
+                for r in ship_rows:
+                    exists = self.session.execute(select(t_ship.c.name).where(t_ship.c.ship_id == r["ship_id"])).first()
+                    if exists is None:
+                        self.session.execute(insert(t_ship).values(**r))
+                    elif exists[0] != r["name"]:
+                        self.session.execute(update(t_ship).where(t_ship.c.ship_id == r["ship_id"]).values(name=r["name"]))
+        
+        wrote = 0
+        if vendor_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_modified(self.session, t_vendor, rows=vendor_rows,
+                                                key_cols=("ship_id", "station_id"), modified_col="modified", update_cols=())
+                wrote = len(vendor_rows)
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_modified(self.session, t_vendor, rows=vendor_rows,
+                                               key_cols=("ship_id", "station_id"), modified_col="modified", update_cols=())
+                wrote = len(vendor_rows)
+            else:
+                for r in vendor_rows:
+                    ven = self.session.execute(
+                        select(t_vendor.c.modified).where(and_(t_vendor.c.ship_id == r["ship_id"], t_vendor.c.station_id == r["station_id"]))
+                    ).first()
+                    if ven is None:
+                        self.session.execute(insert(t_vendor).values(**r))
+                        wrote += 1
+                    else:
+                        dbm = ven[0]
+                        if dbm is None or r["modified"] > dbm:
+                            self.session.execute(
+                                update(t_vendor)
+                                .where(and_(t_vendor.c.ship_id == r["ship_id"], t_vendor.c.station_id == r["station_id"]))
+                                .values(modified=r["modified"])
+                            )
+                            wrote += 1
+        return wrote
+    
+    def _upsert_outfitting(self, tables: dict[str, Table], station_id: int, modules: list[dict[str, Any]], ts: datetime) -> int:
+        t_up, t_vendor = tables["Upgrade"], tables["UpgradeVendor"]
+        up_rows, vendor_rows = [], []
+        
+        for mo in modules:
+            up_id = mo.get("moduleId")
+            name = mo.get("name")
+            cls = mo.get("class")
+            rating = mo.get("rating")
+            ship = mo.get("ship")
+            if up_id is None or name is None:
+                continue
+            
+            up_rows.append({"upgrade_id": up_id, "name": name, "class": cls, "rating": rating, "ship": ship})
+            vendor_rows.append({"upgrade_id": up_id, "station_id": station_id, "modified": ts})
+        
+        if up_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_simple(self.session, t_up, rows=up_rows, key_cols=("upgrade_id",),
+                                              update_cols=("name", "class", "rating", "ship"))
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_simple(self.session, t_up, rows=up_rows, key_cols=("upgrade_id",),
+                                             update_cols=("name", "class", "rating", "ship"))
+            else:
+                for r in up_rows:
+                    exists = self.session.execute(select(t_up.c.upgrade_id).where(t_up.c.upgrade_id == r["upgrade_id"])).first()
+                    if exists is None:
+                        self.session.execute(insert(t_up).values(**r))
+                    else:
+                        self.session.execute(
+                            update(t_up).where(t_up.c.upgrade_id == r["upgrade_id"]).values(
+                                name=r["name"], **{"class": r["class"]}, rating=r["rating"], ship=r["ship"]
+                            )
+                        )
+        
+        wrote = 0
+        if vendor_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_modified(self.session, t_vendor, rows=vendor_rows,
+                                                key_cols=("upgrade_id", "station_id"), modified_col="modified", update_cols=())
+                wrote = len(vendor_rows)
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_modified(self.session, t_vendor, rows=vendor_rows,
+                                               key_cols=("upgrade_id", "station_id"), modified_col="modified", update_cols=())
+                wrote = len(vendor_rows)
+            else:
+                for r in vendor_rows:
+                    ven = self.session.execute(
+                        select(t_vendor.c.modified).where(and_(t_vendor.c.upgrade_id == r["upgrade_id"], t_vendor.c.station_id == r["station_id"]))
+                    ).first()
+                    if ven is None:
+                        self.session.execute(insert(t_vendor).values(**r))
+                        wrote += 1
+                    else:
+                        dbm = ven[0]
+                        if dbm is None or r["modified"] > dbm:
+                            self.session.execute(
+                                update(t_vendor)
+                                .where(and_(t_vendor.c.upgrade_id == r["upgrade_id"], t_vendor.c.station_id == r["station_id"]))
+                                .values(modified=r["modified"])
+                            )
+                            wrote += 1
+        return wrote
+    
+    def _upsert_market(
+        self,
+        tables: dict[str, Table],
+        categories: dict[str, int],
+        station_id: int,
+        commodities: list[dict[str, Any]],
+        ts: datetime,
+    ) -> tuple[int, int]:
+        t_item, t_si = tables["Item"], tables["StationItem"]
+        item_rows, link_rows = [], []
+        wrote_items = 0
+        
+        for co in commodities:
+            fdev_id = co.get("commodityId")
+            name = co.get("name")
+            cat_name = co.get("category")
+            if fdev_id is None or name is None or cat_name is None:
+                continue
+            
+            cat_id = categories.get(str(cat_name).lower())
+            if cat_id is None:
+                raise CleanExit(f'Unknown commodity category "{cat_name}"')
+            
+            item_rows.append({"item_id": fdev_id, "name": name, "category_id": cat_id, "fdev_id": fdev_id, "ui_order": 0})
+            
+            demand = co.get("demand")
+            supply = co.get("supply")
+            buy = co.get("buyPrice")
+            sell = co.get("sellPrice")
+            
+            link_rows.append({
+                "station_id":   station_id,
+                "item_id":      fdev_id,
+                "demand_price": sell,
+                "demand_units": demand,
+                "demand_level": -1,
+                "supply_price": buy,
+                "supply_units": supply,
+                "supply_level": -1,
+                "from_live":    0,
+                "modified":     ts,
+            })
+        
+        if item_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_simple(self.session, t_item, rows=item_rows, key_cols=("item_id",),
+                                              update_cols=("name", "category_id", "fdev_id", "ui_order"))
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_simple(self.session, t_item, rows=item_rows, key_cols=("item_id",),
+                                             update_cols=("name", "category_id", "fdev_id", "ui_order"))
+            else:
+                for r in item_rows:
+                    exists = self.session.execute(
+                        select(t_item.c.item_id, t_item.c.name, t_item.c.category_id).where(t_item.c.item_id == r["item_id"])
+                    ).first()
+                    if exists is None:
+                        self.session.execute(insert(t_item).values(**r))
+                        wrote_items += 1
+                    else:
+                        _, db_name, db_cat = exists
+                        if (db_name != r["name"]) or (db_cat != r["category_id"]):
+                            self.session.execute(
+                                update(t_item).where(t_item.c.item_id == r["item_id"]).values(
+                                    name=r["name"], category_id=r["category_id"]
+                                )
+                            )
+        
+        wrote_links = 0
+        if link_rows:
+            if db_utils.is_sqlite(self.session):
+                db_utils.sqlite_upsert_modified(self.session, t_si, rows=link_rows,
+                                                key_cols=("station_id", "item_id"), modified_col="modified",
+                                                update_cols=("demand_price", "demand_units", "demand_level",
+                                                             "supply_price", "supply_units", "supply_level", "from_live"))
+                wrote_links = len(link_rows)
+            elif db_utils.is_mysql(self.session):
+                db_utils.mysql_upsert_modified(self.session, t_si, rows=link_rows,
+                                               key_cols=("station_id", "item_id"), modified_col="modified",
+                                               update_cols=("demand_price", "demand_units", "demand_level",
+                                                            "supply_price", "supply_units", "supply_level", "from_live"))
+                wrote_links = len(link_rows)
+            else:
+                for r in link_rows:
+                    si = self.session.execute(
+                        select(t_si.c.modified).where(and_(t_si.c.station_id == r["station_id"], t_si.c.item_id == r["item_id"]))
+                    ).first()
+                    if si is None:
+                        self.session.execute(insert(t_si).values(**r))
+                        wrote_links += 1
+                    else:
+                        dbm = si[0]
+                        if dbm is None or r["modified"] > dbm:
+                            self.session.execute(
+                                update(t_si)
+                                .where(and_(t_si.c.station_id == r["station_id"], t_si.c.item_id == r["item_id"]))
+                                .values(**r)
+                            )
+                            wrote_links += 1
+        
+        return (wrote_items, wrote_links)
+    
+    # ------------------------------
+    # UI ordering
+    # ------------------------------
+    def _enforce_ui_order(self, session: Session, tables: dict[str, Table]) -> None:
+        t_item, t_cat = tables["Item"], tables["Category"]
+        cats = session.execute(select(t_cat.c.category_id)).all()
+        for (cat_id,) in cats:
+            rows = session.execute(
+                select(t_item.c.item_id, t_item.c.name, t_item.c.ui_order)
+                .where(t_item.c.category_id == cat_id)
+                .order_by(func.lower(t_item.c.name).asc(), t_item.c.name.asc(), t_item.c.item_id.asc())
+            ).all()
+            expected = 1
+            for item_id, _name, ui_order in rows:
+                if ui_order != expected:
+                    session.execute(update(t_item).where(t_item.c.item_id == item_id).values(ui_order=expected))
+                expected += 1
+    
+    # ------------------------------
+    # Rares import (via cache.processImportFile)
+    # ------------------------------
+    def _import_rareitems_edcd(self, rares_csv: Path, commodity_csv: Optional[Path] = None) -> None:
+        """
+        EDCD rares → TD.RareItem
+        
+        Supports CSV shapes:
+          A) name, system, station
+          B) id, symbol, market_id, category, name  (FDevIDs canonical)
+        
+        Shape B maps: station_id = int(market_id), category by name.
+        Clears RareItem then upserts by UNIQUE(name). Writes a CSV of skipped rows to tmp/.
+        """
+        
+        def _norm(s: Optional[str]) -> str:
+            if s is None:
+                return ""
+            s = s.strip().strip("'").strip('"')
+            s = s.replace("’", "'").replace("‘", "'")
+            s = s.replace("–", "-").replace("—", "-")
+            s = " ".join(s.split())
+            return s.casefold()
+        
+        def _kwant(fieldnames, *aliases) -> Optional[str]:
+            if not fieldnames:
+                return None
+            canon = {}
+            for h in fieldnames or []:
+                if not h:
+                    continue
+                k = h.strip().lower().replace("_", "").replace(" ", "")
+                canon[k] = h
+            for a in aliases:
+                k = a.strip().lower().replace("_", "").replace(" ", "")
+                if k in canon:
+                    return canon[k]
+            return None
+        
+        sess = None
+        try:
+            sess = self._open_session()
+            tables = self._reflect_tables(sess.get_bind())
+            t_sys, t_stn, t_cat, t_rare = tables["System"], tables["Station"], tables["Category"], tables["RareItem"]
+            
+            # Build lookups for Shape A
+            stn_by_names: dict[tuple[str, str], int] = {}
+            for sid, sys_name, stn_name in sess.execute(
+                select(t_stn.c.station_id, t_sys.c.name, t_stn.c.name).where(t_stn.c.system_id == t_sys.c.system_id)
+            ).all():
+                if sys_name and stn_name:
+                    stn_by_names[(_norm(sys_name), _norm(stn_name))] = int(sid)
+            
+            # Category name -> id (from DB)
+            cat_id_by_name = {
+                _norm(n): int(cid)
+                for cid, n in sess.execute(select(t_cat.c.category_id, t_cat.c.name)).all()
+                if n is not None
+            }
+            
+            kept = skipped = 0
+            skipped_no_station = 0
+            skipped_no_category = 0
+            out_rows: list[dict] = []
+            skipped_rows: list[dict] = []   # <-- record details
+            
+            with open(rares_csv, "r", encoding="utf-8", newline="") as fh:
+                reader = csv.DictReader(fh)
+                hdr = [h for h in (reader.fieldnames or []) if h]
+                hdr_canon = [h.lower().replace("_", "").replace(" ", "") for h in hdr]
+                
+                has_market_shape = all(x in hdr_canon for x in ["id", "symbol", "marketid", "category", "name"])
+                has_name_shape   = all(x in hdr_canon for x in ["name", "system", "station"])
+                
+                if not (has_market_shape or has_name_shape):
+                    raise CleanExit(
+                        "rare_commodity.csv headers not recognized. "
+                        f"Seen headers: {', '.join(reader.fieldnames or [])}. File: {rares_csv}"
+                    )
+                
+                if has_market_shape:
+                    # FDevIDs: station_id = int(market_id)
+                    k_name   = _kwant(reader.fieldnames, "name")
+                    k_market = _kwant(reader.fieldnames, "market_id", "marketid")
+                    k_cat    = _kwant(reader.fieldnames, "category", "categoryname")
+                    
+                    for row in reader:
+                        rn_raw = row.get(k_name)
+                        mk_raw = row.get(k_market)
+                        cat_raw= row.get(k_cat)
+                        
+                        try:
+                            station_id = int(mk_raw) if mk_raw is not None else None
+                        except (TypeError, ValueError):
+                            station_id = None
+                        
+                        # validate station exists
+                        if station_id is None or sess.execute(
+                            select(t_stn.c.station_id).where(t_stn.c.station_id == station_id)
+                        ).first() is None:
+                            skipped += 1
+                            skipped_no_station += 1
+                            skipped_rows.append({"reason":"no_station","name":rn_raw,"market_id":mk_raw,"category":cat_raw})
+                            continue
+                        
+                        cid = cat_id_by_name.get(_norm(cat_raw))
+                        if cid is None:
+                            skipped += 1
+                            skipped_no_category += 1
+                            skipped_rows.append({"reason":"no_category","name":rn_raw,"market_id":mk_raw,"category":cat_raw})
+                            continue
+                        
+                        out_rows.append({
+                            "name": rn_raw,
+                            "station_id": station_id,
+                            "category_id": cid,
+                            "cost": None,
+                            "max_allocation": None,
+                        })
+                        kept += 1
+                
+                else:
+                    # Legacy/community: need commodity.csv to map product -> category
+                    name_to_catid: dict[str, int] = {}
+                    if commodity_csv is None:
+                        files = self._acquire_edcd_files()
+                        commodity_csv = files.get("commodity")
+                    if commodity_csv and Path(commodity_csv).exists():
+                        with open(commodity_csv, "r", encoding="utf-8", newline="") as fh2:
+                            rd2 = csv.DictReader(fh2)
+                            k2_name = _kwant(rd2.fieldnames, "name","commodity","commodityname","product")
+                            k2_cat  = _kwant(rd2.fieldnames, "category","categoryname")
+                            if k2_name and k2_cat:
+                                for r2 in rd2:
+                                    n = _norm(r2.get(k2_name))
+                                    c = _norm(r2.get(k2_cat))
+                                    if n and c:
+                                        cid = cat_id_by_name.get(c)
+                                        if cid is not None:
+                                            name_to_catid[n] = cid
+                    
+                    k_name    = _kwant(reader.fieldnames, "name","commodity","commodityname","product")
+                    k_system  = _kwant(reader.fieldnames, "system","systemname")
+                    k_station = _kwant(reader.fieldnames, "station","stationname")
+                    
+                    for row in reader:
+                        rn_raw  = row.get(k_name)
+                        sys_raw = row.get(k_system)
+                        stn_raw = row.get(k_station)
+                        rn, sysn, stnn = _norm(rn_raw), _norm(sys_raw), _norm(stn_raw)
+                        
+                        if not rn or not sysn or not stnn:
+                            skipped += 1
+                            skipped_rows.append({"reason":"missing_fields","name":rn_raw,"system":sys_raw,"station":stn_raw})
+                            continue
+                        
+                        station_id = stn_by_names.get((sysn, stnn))
+                        if station_id is None:
+                            skipped += 1
+                            skipped_no_station += 1
+                            skipped_rows.append({"reason":"no_station","name":rn_raw,"system":sys_raw,"station":stn_raw})
+                            continue
+                        
+                        cid = name_to_catid.get(rn)
+                        if cid is None:
+                            skipped += 1
+                            skipped_no_category += 1
+                            skipped_rows.append({"reason":"no_category","name":rn_raw,"system":sys_raw,"station":stn_raw})
+                            continue
+                        
+                        out_rows.append({
+                            "name": rn_raw,
+                            "station_id": station_id,
+                            "category_id": cid,
+                            "cost": None,
+                            "max_allocation": None,
+                        })
+                        kept += 1
+            
+            # Clear → upsert
+            try:
+                sess.execute(text('DELETE FROM "RareItem"'))
+            except Exception:
+                sess.execute(text("DELETE FROM RareItem"))
+            
+            if out_rows:
+                if db_utils.is_sqlite(sess):
+                    db_utils.sqlite_upsert_simple(
+                        sess, t_rare, rows=out_rows, key_cols=("name",),
+                        update_cols=tuple(k for k in out_rows[0].keys() if k != "name")
+                    )
+                elif db_utils.is_mysql(sess):
+                    db_utils.mysql_upsert_simple(
+                        sess, t_rare, rows=out_rows, key_cols=("name",),
+                        update_cols=tuple(k for k in out_rows[0].keys() if k != "name")
+                    )
+                else:
+                    for r in out_rows:
+                        ex = sess.execute(select(t_rare.c.name).where(t_rare.c.name == r["name"])).first()
+                        if ex is None:
+                            sess.execute(insert(t_rare).values(**r))
+                        else:
+                            sess.execute(
+                                update(t_rare).where(t_rare.c.name == r["name"])
+                                .values({k: r[k] for k in r.keys() if k != "name"})
+                            )
+            sess.commit()
+            
+            # Write a CSV with skipped details
+            if skipped_rows:
+                outp = self.tmp_dir / "edcd_rares_skipped.csv"
+                keys = sorted({k for r in skipped_rows for k in r.keys()})
+                with open(outp, "w", encoding="utf-8", newline="") as fh:
+                    w = csv.DictWriter(fh, fieldnames=keys)
+                    w.writeheader()
+                    w.writerows(skipped_rows)
+                self._print(f"EDCD Rares: imported={kept:,}  skipped={skipped:,}  "
+                            f"(no_station={skipped_no_station:,}, no_category={skipped_no_category:,})  "
+                            f"→ details: {outp}")
+            else:
+                self._print(f"EDCD Rares: imported={kept:,}  skipped={skipped:,}  "
+                            f"(no_station={skipped_no_station:,}, no_category={skipped_no_category:,})")
+        
+        except Exception as e:
+            if sess is not None:
+                try:
+                    sess.rollback()
+                except Exception:
+                    pass
+            raise CleanExit(f"RareItem import failed: {e!r}") from e
+        finally:
+            if sess is not None:
+                try:
+                    sess.close()
+                except Exception:
+                    pass
+    
+    # ------------------------------
+    # Export / cache refresh
+    #
+    def _export_cache(self) -> None:
+        """
+        Export CSVs and regenerate TradeDangerous.prices — concurrently, with optional StationItem gating.
+        
+        IMPORTANT:
+          - CSV exports are written to tdenv.dataDir (private) so they remain authoritative.
+          - A separate mirror step publishes selected/all CSVs to TD_CSV (public).
+        """
+        
+        def _opt_true(val: Optional[str]) -> bool:
+            if val is None:
+                return False
+            if isinstance(val, str):
+                return val.strip().lower() in ("1", "true", "yes", "on", "y")
+            return bool(val)
+        
+        skip_stationitems = (
+            _opt_true(self.getOption("skip_stationitems"))
+            or _opt_true(os.environ.get("TD_SKIP_STATIONITEM_EXPORT"))
+        )
+        
+        # Export destination: always private dataDir
+        export_dir = Path(self.tdenv.dataDir).resolve()
+        try:
+            export_dir.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            raise CleanExit(f"Export failed: unable to create export directory {export_dir}: {e!r}") from None
+        
+        # Heaviest tables first to maximize overlap
+        tables = [
+            "StationItem",
+            "ShipVendor",
+            "UpgradeVendor",
+            "Station",
+            "System",
+            "Category",          # <-- REQUIRED for correct downstream category mapping
+            "Item",
+            "Ship",
+            "Upgrade",
+            "RareItem",
+            "FDevOutfitting",
+            "FDevShipyard",
+        ]
+        if skip_stationitems:
+            tables = [t for t in tables if t != "StationItem"]
+        
+        # Worker count (env override allowed); +1 slot reserved for prices task
+        try:
+            workers = int(os.environ.get("TD_EXPORT_WORKERS", "4"))
+        except ValueError:
+            workers = 4
+        workers = max(1, workers) + 1  # extra slot for the prices job
+        
+        def _export_one(table_name: str) -> str:
+            sess = None
+            try:
+                sess = self._open_session()  # fresh session per worker
+                csvexport.exportTableToFile(sess, self.tdenv, table_name, csvPath=export_dir)
+                return f"{table_name}.csv"
+            finally:
+                if sess is not None:
+                    try:
+                        sess.close()
+                    except Exception:
+                        pass
+        
+        def _regen_prices() -> str:
+            cache.regeneratePricesFile(self.tdb, self.tdenv)
+            return "TradeDangerous.prices"
+        
+        self._print(f"Exporting cache CSVs to: {export_dir}")
+        for t in tables:
+            self._print(f"  - {t}.csv")
+        if skip_stationitems:
+            self._warn("Skipping StationItem.csv export (requested).")
+        self._print("Regenerating TradeDangerous.prices …")
+        
+        # Parallel export + prices regen, with conservative fallback
+        try:
+            with ThreadPoolExecutor(max_workers=workers) as ex:
+                futures = {ex.submit(_export_one, t): f"{t}.csv" for t in tables}
+                futures[ex.submit(_regen_prices)] = "TradeDangerous.prices"
+                for fut in as_completed(futures):
+                    _ = fut.result()  # raise on any worker failure
+        except Exception as e:
+            self._warn(f"Parallel export encountered an error ({e!r}); falling back to serial.")
+            for t in tables:
+                _export_one(t)
+            _regen_prices()
+        
+        self._print("Cache export completed.")
+    
+    def _mirror_csv_exports(self) -> None:
+        """
+        If TD_CSV is set, mirror all CSVs emitted into tdenv.dataDir to TD_CSV.
+        
+        This is a publish step:
+          - source: private exports in tdenv.dataDir (TD_DATA)
+          - dest:   public directory TD_CSV
+        """
+        src_dir = Path(self.tdenv.dataDir).resolve()
+        dst_env = os.environ.get("TD_CSV")
+        if not dst_env:
+            return
+        dst_dir = Path(dst_env).expanduser().resolve()
+        
+        if src_dir == dst_dir:
+            # Nothing to do; already exporting directly into the public path
+            return
+        
+        try:
+            dst_dir.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            self._warn(f"TD_CSV mirror: unable to create destination {dst_dir}: {e!r}")
+            return
+        
+        copied = 0
+        for src in src_dir.glob("*.csv"):
+            try:
+                shutil.copy2(src, dst_dir / src.name)
+                copied += 1
+            except Exception as e:
+                self._warn(f"TD_CSV mirror: failed to copy {src.name}: {e!r}")
+        
+        self._print(f"TD_CSV mirror: copied {copied} csv file(s) → {dst_dir}")
+    
+    
+    def _export_and_mirror(self) -> None:
+        """
+        Run the normal cache/CSV export, then mirror CSVs to TD_CSV if set.
+        Use this in place of a direct _export_cache() call.
+        """
+        t0 = time.time()
+        self._export_cache()  # existing exporter (unchanged)
+        self._print(f"Cache export completed in {time.time()-t0:.2f}s")
+        self._mirror_csv_exports()
+    
+    # ------------------------------
+    # Categories cache
+    #
+    def _load_categories(self, session: Session, tables: dict[str, Table]) -> dict[str, int]:
+        t_cat = tables["Category"]
+        rows = session.execute(select(t_cat.c.category_id, t_cat.c.name)).all()
+        return {str(name).lower(): int(cid) for (cid, name) in rows}
+    
+    # ------------------------------
+    # Streaming JSON reader
+    #
+    def _ijson_items(self, fh: io.BufferedReader, prefix: str):
+        """
+        Use the fastest available ijson backend with clean fallback.
+        Order: yajl2_cffi → yajl2_c → yajl2 → python.
+        """
+        try:
+            from ijson.backends import yajl2_cffi as ijson_fast
+            return ijson_fast.items(fh, prefix)
+        except Exception:
+            pass
+        try:
+            from ijson.backends import yajl2_c as ijson_fast  # ctypes wrapper
+            return ijson_fast.items(fh, prefix)
+        except Exception:
+            pass
+        try:
+            from ijson.backends import yajl2 as ijson_fast
+            return ijson_fast.items(fh, prefix)
+        except Exception:
+            pass
+        # Fallback to whatever was imported at module top
+        return ijson.items(fh, prefix)
+    
+    def _iter_top_level_json_array(self, fh: io.BufferedReader) -> Generator[dict[str, Any]]:
+        """
+        High-performance streaming reader for a huge top-level JSON array of systems.
+        NOTE: As of 2025-10, we removed _parse_progress(). This iterator now
+        maintains byte/rate metrics only; rendering is handled by _progress_line().
+        """
+        start_ts = time.time()
+        last_tick_systems = 0
+        TICK_EVERY = 256
+        
+        it = self._ijson_items(fh, 'item')
+        for idx, obj in enumerate(it, 1):
+            if (idx - last_tick_systems) >= TICK_EVERY:
+                last_tick_systems = idx
+                # Update parse metrics (no printing here)
+                try:
+                    pos = fh.tell()
+                    elapsed = max(time.time() - start_ts, 1e-9)
+                    self._parse_bytes = pos
+                    self._parse_rate = pos / elapsed
+                except Exception:
+                    pass
+            yield obj
+        
+        # Final metric update at EOF
+        try:
+            pos = fh.tell()
+            elapsed = max(time.time() - start_ts, 1e-9)
+            self._parse_bytes = pos
+            self._parse_rate = pos / elapsed
+        except Exception:
+            pass
+        
+        if self._is_tty:
+            self._live_status("")
+    
+    # ------------------------------
+    # Mapping / derivations / misc
+    #
+    @staticmethod
+    def _build_station_type_map() -> dict[Optional[str], tuple[int, bool]]:
+        return {
+            None: (0, False),
+            "None": (0, False),
+            "Outpost": (1, False),
+            "Coriolis Starport": (2, False),
+            "Ocellus Starport": (3, False),
+            "Orbis Starport": (4, False),
+            "Planetary Outpost": (11, True),
+            "Planetary Port": (12, True),
+            "Mega ship": (13, False),
+            "Asteroid base": (14, False),
+            "Drake-Class Carrier": (24, False),
+            "Settlement": (25, True),
+        }
+    
+    def _map_station_type(self, type_name: Optional[str]) -> tuple[int, str]:
+        if isinstance(type_name, str):
+            res = self._station_type_map.get(type_name)
+            if res:
+                type_id, is_planetary = res
+                return type_id, "Y" if is_planetary else "N"
+        return (0, "?")
+    
+    @staticmethod
+    def _derive_pad_size(landing: Mapping[str, Any]) -> str:
+        try:
+            if landing.get("large"):
+                return "L"
+            if landing.get("medium"):
+                return "M"
+            if landing.get("small"):
+                return "S"
+        except Exception:
+            pass
+        return "?"
+    
+    def _resolve_batch_size(self) -> Optional[int]:
+        """
+        Decide commit batch size for *spansh* profile.
+        """
+        if self.session is not None and hasattr(db_utils, "get_import_batch_size"):
+            try:
+                val = db_utils.get_import_batch_size(self.session, profile="spansh")
+                if val is not None:
+                    return val
+            except Exception:
+                pass
+        
+        raw = os.environ.get("TD_LISTINGS_BATCH")
+        if raw is not None:
+            try:
+                envv = int(raw)
+                return envv if envv > 0 else None
+            except ValueError:
+                pass
+        
+        try:
+            if db_utils.is_sqlite(self.session):
+                return None
+            if db_utils.is_mysql(self.session):
+                return 50_000
+        except Exception:
+            pass
+        
+        return 5_000
+    
+    # ---- ts/format/logging helpers ----
+    def _parse_ts(self, value: Any) -> Optional[datetime]:
+        try:
+            return db_utils.parse_ts(value)  # UTC-naive, μs=0
+        except Exception:
+            return None
+    
+    @staticmethod
+    def _format_hms(seconds: float) -> str:
+        m, s = divmod(int(seconds), 60)
+        h, m = divmod(m, 60)
+        return f"{h}:{m:02d}:{s:02d}"
+    
+    def _fmt_bytes(self, n: float) -> str:
+        units = ["B", "KiB", "MiB", "GiB", "TiB"]
+        i = 0
+        while n >= 1024 and i < len(units) - 1:
+            n /= 1024.0
+            i += 1
+        return f"{int(n)} {units[i]}" if i == 0 else f"{n:.1f} {units[i]}"
+    
+    def _progress_line(self, stats: dict[str, int]) -> None:
+        """
+        Single-line live status while importing.
+        
+        Modes:
+          - default (verbose-ish): rich long line
+          - compact: shorter, log-friendly line (enable with -O progress_compact=1 or TD_PROGRESS_COMPACT=1)
+        """
+        now = time.time()
+        if now - self._last_progress_time < (0.5 if self._debug_level < 1 else 0.2):
+            return
+        self._last_progress_time = now
+        self._started_importing = True
+        
+        # Determine compact mode (CLI overrides env; default is rich/False)
+        # Truthy whitelist: 1, true, yes, on, y (case-insensitive)
+        _opt = self.getOption("progress_compact")
+        if _opt is not None:
+            _val = str(_opt).strip().lower()
+        else:
+            _env = os.getenv("TD_PROGRESS_COMPACT")
+            _val = "" if _env is None else str(_env).strip().lower()
+        compact = _val in {"1", "true", "yes", "on", "y"}
+        
+        parse_bytes = getattr(self, "_parse_bytes", 0)
+        parse_rate  = getattr(self, "_parse_rate", 0.0)
+        systems     = stats.get("systems", 0)
+        stations    = stats.get("stations", 0)
+        
+        wm = stats.get("market_writes", 0)
+        wo = stats.get("outfit_writes", 0)
+        ws = stats.get("ship_writes", 0)
+        
+        km = stats.get("market_stations", 0)
+        ko = stats.get("outfit_stations", 0)
+        ks = stats.get("ship_stations", 0)
+        
+        if compact:
+            # Compact, log-friendly (newline prints)
+            msg = (
+                f"Importing…  {parse_bytes/1048576:.1f} MiB read  {parse_rate/1048576:.1f} MiB/s  "
+                f"systems:{systems:,}  stations:{stations:,}  "
+                f"checked m/o/s:{km:,}/{ko:,}/{ks:,}  written m/o/s:{wm:,}/{wo:,}/{ws:,}"
+            )
+            self._print(msg)
+            return
+        
+        # Rich/long line (TTY-optimized; truncated only on TTY)
+        msg = (
+            f"Importing…  {parse_bytes/1048576:.1f} MiB read  {parse_rate/1048576:.1f} MiB/s  "
+            f"[Parsed - Systems: {systems:,} Stations: {stations:,}]  "
+            f"Checked(stations): mkt={km:,} outf={ko:,} shp={ks:,}  "
+            f"Written(stations): mkt={wm:,} outf={wo:,} shp={ws:,}"
+        
+        )
+        self._live_status(msg)
+    
+    def _live_line(self, msg: str) -> None:
+        self._live_status(msg)
+    
+    def _live_status(self, msg: str) -> None:
+        """
+        Live status line for TTY; plain prints for non-TTY.
+        IMPORTANT: only truncate when TTY so logs are not cut off.
+        """
+        try:
+            if self._is_tty:
+                width = shutil.get_terminal_size(fallback=(120, 20)).columns
+                if width and width > 4:
+                    msg = msg[: width - 2]
+                s = f"\x1b[2K\r{msg}"
+                sys.stderr.write(s)
+                sys.stderr.flush()
+            else:
+                # Non-TTY: emit full line, no truncation, no control codes.
+                self._print(msg)
+        except Exception:
+            self._print(msg)
+    
+    def _end_live_status(self) -> None:
+        try:
+            if self._is_tty:
+                sys.stderr.write("\x1b[2K\r\n")
+                sys.stderr.flush()
+        except Exception:
+            pass
+    
+    # ---- printing/warnings ----
+    def _print(self, *args, **kwargs):
+        printer = getattr(self.tdenv, "print", None)
+        if callable(printer):
+            printer(*args, **kwargs)
+        else:
+            print(*args, **kwargs)
+    
+    def _warn(self, msg: str):
+        if self._warn_enabled:
+            self._print(f"WARNING: {msg}")
+    
+    def _error(self, msg: str):
+        self._print(f"ERROR: {msg}")
+    
+    def _safe_close_session(self):
+        try:
+            if self.session is not None:
+                self.session.close()
+        except Exception:
+            pass
+        self.session = None
+
+# -----------------------------------------------------------------------------
+# Exceptions
+# -----------------------------------------------------------------------------
+class CleanExit(Exception):
+    """Controlled early exit: log and stop this run so schedulers can retry later."""
+    pass