tradedangerous 11.5.3__py3-none-any.whl → 12.0.1__py3-none-any.whl

tradedangerous/csvexport.py

@@ -1,31 +1,31 @@
 from pathlib import Path
+from sqlalchemy import inspect, text
 from .tradeexcept import TradeException
 
+from .db import utils as db_utils
+
+
 import csv
 import os
-import sqlite3
 
 ######################################################################
 # TradeDangerous :: Modules :: CSV Exporter
 #
-# Generate a CSV files for a table of the database.
+# Generate CSV files for database tables.
 #
-# Note: This routine makes some assumptions about the structure
-#       of the database:
-#          * The table should only have one UNIQUE index
-#          * The referenced table must have one UNIQUE index
-#          * The FK columns must have the same name in both tables
-#          * One column primary keys will be handled by the database engine
+# Assumptions:
+#   * Each table has at most one UNIQUE index.
+#   * Referenced tables also have a UNIQUE index.
+#   * Only single-column foreign keys are supported.
+#   * Single-column primary keys are inferred automatically by SQLAlchemy.
 #
-######################################################################
-# CAUTION: If the database structure gets changed this script might
-#          need some corrections.
+# CAUTION: If the schema changes this module may require updates.
 ######################################################################
 
 ######################################################################
 # Default values
 
-# for some tables the first two columns will be reversed
+# For some tables the first two columns will be reversed
 reverseList = []
 
 ######################################################################
@@ -38,183 +38,194 @@ def search_keyList(items, val):
             return row
     return None
 
-def getUniqueIndex(conn, tableName):
-    """ return all unique columns """
-    idxCursor = conn.cursor()
+
+def getUniqueIndex(session, tableName):
+    """Return all unique columns via SQLAlchemy inspector."""
+    inspector = inspect(session.get_bind())
     unqIndex = []
-    for idxRow in idxCursor.execute("PRAGMA index_list('%s')" % tableName):
-        if idxRow['unique']:
-            # it's a unique index
-            unqCursor = conn.cursor()
-            for unqRow in unqCursor.execute("PRAGMA index_info('%s')" % idxRow['name']):
-                unqIndex.append(unqRow['name'])
+    for idx in inspector.get_indexes(tableName):
+        if idx.get("unique"):
+            unqIndex.extend(idx.get("column_names", []))
     return unqIndex
 
-def getFKeyList(conn, tableName):
-    """ get all single column FKs """
+
+def getFKeyList(session, tableName):
+    """Return all single-column foreign keys via SQLAlchemy inspector."""
+    inspector = inspect(session.get_bind())
     keyList = []
-    keyCount = -1
-    keyCursor = conn.cursor()
-    for keyRow in keyCursor.execute("PRAGMA foreign_key_list('%s')" % tableName):
-        if keyRow['seq'] == 0:
-            keyCount += 1
-            keyList.append( {'table': keyRow['table'],
-                             'from': keyRow['from'],
-                             'to': keyRow['to']}
-                          )
-        if keyRow['seq'] == 1:
-            # if there is a second column, remove it from the list
-            keyList.remove( keyList[keyCount] )
-            keyCount -= 1
-    
+    for fk in inspector.get_foreign_keys(tableName):
+        cols = fk.get("constrained_columns", [])
+        referred = fk.get("referred_columns", [])
+        if len(cols) == 1 and len(referred) == 1:
+            keyList.append({
+                "table": fk.get("referred_table"),
+                "from": cols[0],
+                "to": referred[0],
+            })
     return keyList
 
-def buildFKeyStmt(conn, tableName, key):
+
+def buildFKeyStmt(session, tableName, key):
     """
-        resolve the FK constrain with the UNIQUE index
-        multicolumn UNIQUEs are allowed as long as the last one
-        will be a single column one
+    Resolve the FK constraint against the UNIQUE index of the
+    referenced table.
+
+    Multicolumn UNIQUEs are allowed, but only the last column
+    may be treated as a single-column join target.
     """
-    unqIndex = getUniqueIndex(conn, key['table'])
-    keyList  = getFKeyList(conn, key['table'])
+    unqIndex = getUniqueIndex(session, key["table"])
+    keyList = getFKeyList(session, key["table"])
     keyStmt = []
+
     for colName in unqIndex:
-        # check if the column is a foreign key
+        # If this unique column is itself a foreign key, recurse
         keyKey = search_keyList(keyList, colName)
         if keyKey:
-            newStmt = buildFKeyStmt(conn, key['table'], keyKey)
-            for row in newStmt:
-                keyStmt.append(row)
+            keyStmt.extend(buildFKeyStmt(session, key["table"], keyKey))
         else:
             keyStmt.append({
-                'table': tableName,
-                'column': colName,
-                'joinTable': key['table'],
-                'joinColumn': key['to']
+                "table": tableName,
+                "column": colName,
+                "joinTable": key["table"],
+                "joinColumn": key["to"],
             })
-    
+
     return keyStmt
 
+
 ######################################################################
 # Code
 ######################################################################
 
-def exportTableToFile(tdb, tdenv, tableName, csvPath=None):
+def exportTableToFile(tdb_or_session, tdenv, tableName, csvPath=None):
     """
-        Generate the csv file for tableName in csvPath
-        returns lineCount, exportPath
+    Generate the CSV file for tableName in csvPath.
+    Returns (lineCount, exportPath).
+
+    Behaviour:
+    - Prefix unique columns with "unq:".
+    - Foreign keys are exported as "<col>@<joinTable>.<uniqueCol>".
+    - Datetime-like values for 'modified' columns are exported as
+      "YYYY-MM-DD HH:MM:SS" (no microseconds).
+
+    Compatible with either:
+      * a SQLAlchemy Session
+      * a TradeDB wrapper exposing .engine
     """
-    
-    # path for csv file
-    csvPath = csvPath or tdb.csvPath
-    if not csvPath.is_dir():
-        raise TradeException("Save location '{}' not found.".format(str(csvPath)))
-    
-    # connect to the database
-    conn = tdb.getDB()
-    conn.row_factory = sqlite3.Row
-    
-    # prefix for unique/ignore columns
+    from sqlalchemy.orm import Session
+
+    # --- Resolve a SQLAlchemy session ---
+    if hasattr(tdb_or_session, "engine"):
+        # Likely a TradeDB instance
+        engine = tdb_or_session.engine
+        session = Session(engine)
+    elif hasattr(tdb_or_session, "get_bind"):
+        # Already a Session
+        session = tdb_or_session
+    else:
+        raise TradeException(
+            f"Unsupported DB object passed to exportTableToFile: {type(tdb_or_session)}"
+        )
+
+    csvPath = csvPath or Path(tdenv.csvDir)
+    if not Path(csvPath).is_dir():
+        raise TradeException(f"Save location '{csvPath}' not found.")
+
     uniquePfx = "unq:"
-    ignorePfx = "!"
-    
-    # create CSV files
-    exportPath = (csvPath / Path(tableName)).with_suffix(".csv")
-    tdenv.DEBUG0("Export Table '{table}' to '{file}'".format(
-                    table=tableName, file=str(exportPath)
-                    ))
-    
+    exportPath = (Path(csvPath) / Path(tableName)).with_suffix(".csv")
+    tdenv.DEBUG0(f"Export Table '{tableName}' to '{exportPath}'")
+
+    def _fmt_ts(val):
+        if hasattr(val, "strftime"):
+            try:
+                return val.strftime("%Y-%m-%d %H:%M:%S")
+            except Exception:
+                pass
+        if isinstance(val, str):
+            s = val
+            if len(s) >= 19 and s[10] == "T":
+                s = s[:10] + " " + s[11:]
+            if len(s) >= 19 and s[4] == "-" and s[7] == "-" and s[10] == " " and s[13] == ":" and s[16] == ":":
+                return s[:19]
+        return val
+
     lineCount = 0
-    with exportPath.open("w", encoding='utf-8', newline="\n") as exportFile:
-        exportOut = csv.writer(exportFile, delimiter=",", quotechar="'", doublequote=True, quoting=csv.QUOTE_NONNUMERIC, lineterminator="\n")
-        
-        cur = conn.cursor()
-        
-        # check for single PRIMARY KEY
-        pkCount = 0
-        for columnRow in cur.execute("PRAGMA table_info('%s')" % tableName):
-            # count the columns of the primary key
-            if columnRow['pk'] > 0:
-                pkCount += 1
-        
-        # build column list
-        columnList = []
-        for columnRow in cur.execute("PRAGMA table_info('%s')" % tableName):
-            # if there is only one PK column, ignore it
-            # if columnRow['pk'] > 0 and pkCount == 1: continue
-            columnList.append(columnRow)
-        
-        if len(columnList) == 0:
-            raise TradeException("No columns to export for table '{}'.".format(tableName))
-        
-        # reverse the first two columns for some tables
-        if tableName in reverseList:
-            columnList[0], columnList[1] = columnList[1], columnList[0]
-        
-        # initialize helper lists
-        csvHead    = []
+    with exportPath.open("w", encoding="utf-8", newline="\n") as exportFile:
+        exportOut = csv.writer(
+            exportFile,
+            delimiter=",",
+            quotechar="'",
+            doublequote=True,
+            quoting=csv.QUOTE_NONNUMERIC,
+            lineterminator="\n",
+        )
+
+        bind = session.get_bind()
+        inspector = inspect(bind)
+
+        try:
+            unique_cols = db_utils.get_unique_columns(session, tableName)
+            fk_list = db_utils.get_foreign_keys(session, tableName)
+        except Exception as e:
+            raise TradeException(f"Failed to introspect table '{tableName}': {e!r}")
+
+        csvHead = []
         stmtColumn = []
-        stmtTable  = [ tableName ]
-        stmtOrder  = []
-        unqIndex   = getUniqueIndex(conn, tableName)
-        keyList    = getFKeyList(conn, tableName)
-        
-        tdenv.DEBUG1('UNIQUE: ' + ", ".join(unqIndex))
-        
-        # iterate over all columns of the table
-        for col in columnList:
-            # check if the column is a foreign key
-            key = search_keyList(keyList, col['name'])
-            if key:
-                # make the join statement
-                keyStmt = buildFKeyStmt(conn, tableName, key)
-                for keyRow in keyStmt:
-                    tdenv.DEBUG1('FK-Stmt: {}'.format(list(keyRow)))
-                    # is the join for the same table
-                    if keyRow['table'] == tableName:
-                        csvPfx = ''
-                        joinStmt = 'USING({})'.format(keyRow['joinColumn'])
-                    else:
-                        # this column must be ignored by the importer, it's only
-                        # used to resolve the FK relation
-                        csvPfx = ignorePfx
-                        joinStmt = 'ON {}.{} = {}.{}'.format(keyRow['table'], keyRow['joinColumn'], keyRow['joinTable'], keyRow['joinColumn'])
-                    if col['name'] in unqIndex:
-                        # column is part of an unique index
-                        csvPfx = uniquePfx + csvPfx
-                    csvHead += [ "{}{}@{}.{}".format(csvPfx, keyRow['column'], keyRow['joinTable'], keyRow['joinColumn']) ]
-                    stmtColumn += [ "{}.{}".format(keyRow['joinTable'], keyRow['column']) ]
-                    if col['notnull']:
-                        stmtTable += [ 'INNER JOIN {} {}'.format(keyRow['joinTable'], joinStmt) ]
-                    else:
-                        stmtTable += [ 'LEFT OUTER JOIN {} {}'.format(keyRow['joinTable'], joinStmt) ]
-                    stmtOrder += [ "{}.{}".format(keyRow['joinTable'], keyRow['column']) ]
+        stmtTable = [tableName]
+        stmtOrder = []
+        is_modified_col = []
+
+        for col in inspector.get_columns(tableName):
+            col_name = col["name"]
+            fk = next((fk for fk in fk_list if fk["from"] == col_name), None)
+            if fk:
+                joinTable = fk["table"]
+                joinColumn = fk["to"]
+                join_unique_cols = db_utils.get_unique_columns(session, joinTable)
+                if not join_unique_cols:
+                    raise TradeException(
+                        f"No unique column found in referenced table '{joinTable}'"
+                    )
+                export_col = join_unique_cols[0]
+                csvPfx = uniquePfx if col_name in unique_cols else ""
+                csvHead.append(f"{csvPfx}{col_name}@{joinTable}.{export_col}")
+                stmtColumn.append(f"{joinTable}.{export_col}")
+                is_modified_col.append(export_col == "modified")
+                nullable = bool(col.get("nullable", True))
+                join_type = "LEFT OUTER JOIN" if nullable else "INNER JOIN"
+                stmtTable.append(
+                    f"{join_type} {joinTable} ON {tableName}.{col_name} = {joinTable}.{joinColumn}"
+                )
+                stmtOrder.append(f"{joinTable}.{export_col}")
             else:
-                # ordinary column
-                if col['name'] in unqIndex:
-                    # column is part of an unique index
-                    csvHead += [ uniquePfx + col['name'] ]
-                    stmtOrder += [ "{}.{}".format(tableName, col['name']) ]
+                if col_name in unique_cols:
+                    csvHead.append(uniquePfx + col_name)
+                    stmtOrder.append(f"{tableName}.{col_name}")
                 else:
-                    csvHead += [ col['name'] ]
-                stmtColumn += [ "{}.{}".format(tableName, col['name']) ]
-        
-        # build the SQL statement
-        sqlStmt = "SELECT {} FROM {}".format(",".join(stmtColumn), " ".join(stmtTable))
-        if len(stmtOrder) > 0:
-            sqlStmt += " ORDER BY {}".format(",".join(stmtOrder))
-        tdenv.DEBUG1("SQL: %s" % sqlStmt)
-        
-        # finally generate the csv file
-        # write header line without quotes
-        exportFile.write("{}\n".format(",".join(csvHead)))
-        for line in cur.execute(sqlStmt):
+                    csvHead.append(col_name)
+                stmtColumn.append(f"{tableName}.{col_name}")
+                is_modified_col.append(col_name == "modified")
+
+        sqlStmt = f"SELECT {','.join(stmtColumn)} FROM {' '.join(stmtTable)}"
+        if stmtOrder:
+            sqlStmt += f" ORDER BY {','.join(stmtOrder)}"
+        tdenv.DEBUG1(f"SQL: {sqlStmt}")
+
+        exportFile.write(f"{','.join(csvHead)}\n")
+
+        for row in session.execute(text(sqlStmt)):
             lineCount += 1
-            tdenv.DEBUG2("{count}: {values}".format(count=lineCount, values=list(line)))
-            exportOut.writerow(list(line))
-        tdenv.DEBUG1("{count} {table}s exported".format(count=lineCount, table=tableName))
-    
-    # Update the DB file so we don't regenerate it.
-    os.utime(str(tdb.dbPath))
-    
+            row_out = [
+                _fmt_ts(val) if is_modified_col[i] else val
+                for i, val in enumerate(row)
+            ]
+            tdenv.DEBUG2(f"{lineCount}: {row_out}")
+            exportOut.writerow(row_out)
+
+        tdenv.DEBUG1(f"{lineCount} {tableName}s exported")
+
+    # Close session if we created it
+    if hasattr(tdb_or_session, "engine"):
+        session.close()
+
     return lineCount, exportPath

tradedangerous/db/__init__.py

@@ -0,0 +1,27 @@
+"""TradeDangerous SQLAlchemy bootstrap package (Stage 3A, Part 1).
+
+Side-effect free on import. Provides a minimal, cross-platform API
+for config loading, path resolution, and engine/session bootstrap.
+
+Usage:
+    from tradedangerous.db import (
+        load_config, resolve_data_dir, resolve_tmp_dir, ensure_dir, resolve_db_config_path,
+        make_engine_from_config, get_session_factory, healthcheck,
+    )
+"""
+from .config import load_config
+from .paths import resolve_data_dir, resolve_tmp_dir, ensure_dir, resolve_db_config_path
+from .engine import make_engine_from_config, get_session_factory, healthcheck
+from .lifecycle import ensure_fresh_db
+
+__all__ = [
+    "load_config",
+    "resolve_data_dir",
+    "resolve_tmp_dir",
+    "ensure_dir",
+    "resolve_db_config_path",
+    "make_engine_from_config",
+    "get_session_factory",
+    "healthcheck",
+    "ensure_fresh_db",
+]

tradedangerous/db/adapter.py

@@ -0,0 +1,191 @@
+# tradedangerous/db/adapter.py
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Dict, Generator, Iterable, Optional, Tuple
+
+from sqlalchemy import select, func
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+# Local engine + ORM (authoritative)
+from .engine import make_engine_from_config, get_session_factory  # uses env/CWD-resolved db_config.ini by default
+from .orm_models import System, Station, Item, StationItem  # canonical models
+from .paths import resolve_db_config_path
+
+# ---- Public factory ---------------------------------------------------------
+
+def get_adapter_if_enabled(cfg_path: Optional[str] = None) -> "TradeDBReadAdapter | None":
+    """
+    Return an adapter when [database] backend != 'sqlite', else None.
+    - No engine/session created at import: construction is lazy.
+    - This is called by tradedb.py (thin gate).
+    """
+    import configparser, os
+    if cfg_path is None:
+        cfg_path = str(resolve_db_config_path())
+    cfg = configparser.ConfigParser()
+    if not os.path.exists(cfg_path):
+        return None
+    with open(cfg_path, "r", encoding="utf-8") as fh:
+        cfg.read_file(fh)
+    backend = (cfg.get("database", "backend", fallback="sqlite") or "sqlite").strip().lower()
+    if backend == "sqlite":
+        return None
+
+    # Engine is created lazily via the property below to honour "no side-effects at import".
+    return TradeDBReadAdapter(cfg_path)
+
+# ---- Adapter (read-only) ----------------------------------------------------
+
+class TradeDBReadAdapter:
+    """
+    Very small, read-only façade over SQLAlchemy for legacy TradeDB reads:
+      - systems() list
+      - lookup system by name (case-insensitive)
+      - station by (system_id, station_name) (case-insensitive)
+      - average selling/buying prices (used by trade_cmd at detail>1)
+    """
+    def __init__(self, cfg_path: str):
+        self._cfg_path = cfg_path
+        self._engine: Optional[Engine] = None
+        self._Session = None  # sessionmaker
+
+    # Lazy engine/session factory (no import-time work)
+    @property
+    def Session(self):
+        if self._Session is None:
+            engine = make_engine_from_config(self._cfg_path)
+            self._engine = engine
+            self._Session = get_session_factory(engine)
+        return self._Session
+
+    @contextmanager
+    def session(self) -> Generator[Session, None, None]:
+        Session = self.Session
+        with Session() as s:
+            yield s
+
+    # ---- Reads mapped to ORM ------------------------------------------------
+
+    def list_system_rows(self) -> Iterable[Tuple[int, str, float, float, float, Optional[int]]]:
+        """
+        Shape matches legacy _loadSystems SELECT:
+        (system_id, name, pos_x, pos_y, pos_z, added_id)
+        """
+        with self.session() as s:
+            rows = s.execute(
+                select(
+                    System.system_id,
+                    System.name,
+                    System.pos_x,
+                    System.pos_y,
+                    System.pos_z,
+                    System.added_id,
+                )
+            )
+            for r in rows:
+                yield (r.system_id, r.name, r.pos_x, r.pos_y, r.pos_z, r.added_id)
+
+    def system_by_name(self, name_ci: str) -> Optional[Tuple[int, str, float, float, float, Optional[int]]]:
+        """
+        Case-insensitive name match for System.
+        """
+        with self.session() as s:
+            row = s.execute(
+                select(
+                    System.system_id, System.name, System.pos_x, System.pos_y, System.pos_z, System.added_id
+                ).where(func.upper(System.name) == func.upper(func.trim(func.cast(name_ci, System.name.type))))
+            ).first()
+            if not row:
+                return None
+            return (row.system_id, row.name, row.pos_x, row.pos_y, row.pos_z, row.added_id)
+
+    def station_by_system_and_name(
+        self, system_id: int, station_name_ci: str
+    ) -> Optional[Tuple[int, int, str, int, str, str, str, str, str, str, str, str, str, int]]:
+        """
+        Return the single Station row by system + name (CI).
+        Shape matches legacy _loadStations row consumed by Station(...):
+          (station_id, system_id, name,
+           ls_from_star, market, blackmarket, shipyard,
+           max_pad_size, outfitting, rearm, refuel, repair, planetary, type_id)
+        """
+        with self.session() as s:
+            r = s.execute(
+                select(
+                    Station.station_id,
+                    Station.system_id,
+                    Station.name,
+                    Station.ls_from_star,
+                    Station.market,
+                    Station.blackmarket,
+                    Station.shipyard,
+                    Station.max_pad_size,
+                    Station.outfitting,
+                    Station.rearm,
+                    Station.refuel,
+                    Station.repair,
+                    Station.planetary,
+                    Station.type_id,
+                ).where(
+                    Station.system_id == system_id,
+                    func.upper(Station.name) == func.upper(func.trim(func.cast(station_name_ci, Station.name.type))),
+                )
+            ).first()
+            if not r:
+                return None
+            return (
+                r.station_id,
+                r.system_id,
+                r.name,
+                r.ls_from_star,
+                r.market,
+                r.blackmarket,
+                r.shipyard,
+                r.max_pad_size,
+                r.outfitting,
+                r.rearm,
+                r.refuel,
+                r.repair,
+                r.planetary,
+                r.type_id,
+            )
+
+    def average_selling(self) -> Dict[int, int]:
+        """
+        {item_id: avg_supply_price>0}
+        Mirrors the legacy SQL used in TradeDB.getAverageSelling.
+        """
+        with self.session() as s:
+            rows = s.execute(
+                select(
+                    Item.item_id,
+                    func.IFNULL(func.avg(StationItem.supply_price), 0),
+                )
+                .select_from(Item.__table__.outerjoin(
+                    StationItem, (Item.item_id == StationItem.item_id) & (StationItem.supply_price > 0)
+                ))
+                .where(StationItem.supply_price > 0)
+                .group_by(Item.item_id)
+            )
+            return {int(item_id): int(avg_cr) for (item_id, avg_cr) in rows}
+
+    def average_buying(self) -> Dict[int, int]:
+        """
+        {item_id: avg_demand_price>0}
+        Mirrors the legacy SQL used in TradeDB.getAverageBuying.
+        """
+        with self.session() as s:
+            rows = s.execute(
+                select(
+                    Item.item_id,
+                    func.IFNULL(func.avg(StationItem.demand_price), 0),
+                )
+                .select_from(Item.__table__.outerjoin(
+                    StationItem, (Item.item_id == StationItem.item_id) & (StationItem.demand_price > 0)
+                ))
+                .where(StationItem.demand_price > 0)
+                .group_by(Item.item_id)
+            )
+            return {int(item_id): int(avg_cr) for (item_id, avg_cr) in rows}