tradedangerous 11.2.0__py3-none-any.whl → 11.3.0__py3-none-any.whl

tradedangerous/plugins/eddblink_plug.py

@@ -4,19 +4,18 @@ https://elite.tromador.com/ to update the Database.
 """
 from __future__ import annotations
 
+from email.utils import parsedate_to_datetime
 from pathlib import Path
+from .. fs import file_line_count
 from .. import plugins, cache, transfers
 from ..misc import progress as pbar
 from ..plugins import PluginException
 
-import certifi
 import csv
 import datetime
-from email.utils import parsedate_to_datetime
 import os
 import requests
 import sqlite3
-import ssl
 import typing
 
 
@@ -26,41 +25,12 @@ if typing.TYPE_CHECKING:
 
 # Constants
 BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
-CONTEXT=ssl.create_default_context(cafile=certifi.where())
 
 
 class DecodingError(PluginException):
     pass
 
 
-def _file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
-    """ counts the number of newline characters in a given file. """
-    # Pre-allocate a buffer so we aren't putting pressure on the garbage collector.
-    buf = bytearray(bufsize)
-    
-    # Capture it's counting method, so we don't have to keep looking that up on
-    # large files.
-    counter = buf.count
-    
-    total = 0
-    with from_file.open("rb") as fh:
-        # Capture the 'readinto' method to avoid lookups.
-        reader = fh.readinto
-
-        # read into the buffer and capture the number of bytes fetched,
-        # which will be 'size' until the last read from the file.
-        read = reader(buf)
-        while read == bufsize:  # nominal case for large files
-            total += counter(b'\n')
-            read = reader(buf)
-
-        # when 0 <= read < bufsize we're on the last page of the
-        # file, so we need to take a slice of the buffer, which creates
-        # a new object and thus we also have to lookup count. it's trivial
-        # but if you have to do it 10,000x it's definitely not a rounding error.
-        return total + buf[:read].count(b'\n')
-
-
 def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
     """ Calculates the number of entries in a listing file by counting the lines. """
     if not listings.exists():
@@ -68,7 +38,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
         return 0
     
     tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
-    count = _file_line_count(listings)
+    count = file_line_count(listings)
     if count <= 1:
         if count == 1:
             tdenv.DEBUG0("Listing count of 1 suggests nothing but a header")
@@ -101,7 +71,6 @@ class ImportPlugin(plugins.ImportPluginBase):
     """
     Plugin that downloads data from eddb.
     """
-    
     pluginOptions = {
         'item':         "Update Items using latest file from server. (Implies '-O system,station')",
         'rare':         "Update RareItems using latest file from server. (Implies '-O system,station')",
@@ -118,6 +87,7 @@ class ImportPlugin(plugins.ImportPluginBase):
         'force':        "Force regeneration of selected items even if source file not updated since previous run. "
                         "(Useful for updating Vendor tables if they were skipped during a '-O clean' run.)",
         'purge':        "Remove any empty systems that previously had fleet carriers.",
+        'optimize':     "Optimize ('vacuum') database after processing.",
         'solo':         "Don't download crowd-sourced market data. (Implies '-O skipvend', supercedes '-O all', '-O clean', '-O listings'.)",
     }
     
@@ -197,22 +167,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         db = self.tdb.getDB()
         self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now())
 
-        db.execute("PRAGMA foreign_keys = OFF")
-
-        self.tdenv.DEBUG0("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        db.execute("DROP TABLE IF EXISTS System_copy")
-        db.execute("""CREATE TABLE System_copy AS SELECT * FROM System
-                            WHERE system_id IN (SELECT system_id FROM Station)
-                    """)
-
-        self.tdenv.DEBUG0("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        db.execute("DELETE FROM System")
-        db.execute("INSERT INTO System SELECT * FROM System_copy")
-
-        self.tdenv.DEBUG0("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        db.execute("PRAGMA foreign_keys = ON")
-        db.execute("DROP TABLE IF EXISTS System_copy")
-
+        db.execute("""
+            DELETE FROM System
+             WHERE NOT EXISTS(SELECT 1 FROM Station WHERE Station.system_id = System.system_id)
+        """)
         db.commit()
 
         self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now())
@@ -224,12 +182,16 @@ class ImportPlugin(plugins.ImportPluginBase):
         """
         listings_path = Path(self.dataPath, listings_file).absolute()
         from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
-        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
         
+        self.tdenv.NOTE("Checking listings")
         total = _count_listing_entries(self.tdenv, listings_path)
         if not total:
+            self.tdenv.NOTE("No listings")
             return
 
+        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
+
+        db = self.tdb.getDB()
         stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
         stmt_flush_station   = """DELETE from StationItem WHERE station_id = ?"""
         stmt_add_listing     = """
@@ -246,20 +208,25 @@ class ImportPlugin(plugins.ImportPluginBase):
         """
         
         # Fetch all the items IDS
-        db = self.tdb.getDB()
         item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
         station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
         last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
         
         cur_station = None
+        is_debug = self.tdenv.debug > 0
         self.tdenv.DEBUG0("Processing entries...")
-        with listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
-            prog = pbar.Progress(total, 50)
-
-            cursor: Optional[sqlite3.Cursor] = db.cursor()
+        
+        # Try to find a balance between doing too many commits where we fail
+        # to get any benefits from constructing transactions, and blowing up
+        # the WAL and memory usage by making massive transactions.
+        max_transaction_items, transaction_items = 32 * 1024, 0
+        with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog,\
+              listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
+            cursor = db.cursor()
+            cursor.execute("BEGIN TRANSACTION")
             
             for listing in csv.DictReader(fh):
-                prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}")
+                prog.increment(1)
                 
                 station_id = int(listing['station_id'])
                 if station_id not in station_lookup:
@@ -269,16 +236,21 @@ class ImportPlugin(plugins.ImportPluginBase):
                 
                 if station_id != cur_station:
                     # commit anything from the previous station, get a new cursor
-                    db.commit()
-                    cur_station, skip_station, cursor = station_id, False, db.cursor()
+                    if transaction_items >= max_transaction_items:
+                        cursor.execute("COMMIT")
+                        transaction_items = 0
+                        cursor.execute("BEGIN TRANSACTION")
+                    cur_station, skip_station = station_id, False
                     
                     # Check if listing already exists in DB and needs updated.
                     last_modified: int = int(last_station_update_times.get(station_id, 0))
                     if last_modified:
                         # When the listings.csv data matches the database, update to make from_live == 0.
                         if listing_time == last_modified and not from_live:
-                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
+                            if is_debug:
+                                self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
                             cursor.execute(stmt_unliven_station, (cur_station,))
+                            transaction_items += 1
                             skip_station = True
                             continue
 
@@ -289,8 +261,10 @@ class ImportPlugin(plugins.ImportPluginBase):
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
-                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
+                        if is_debug:
+                            self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
                         cursor.execute(stmt_flush_station, (cur_station,))
+                        transaction_items += 1
                         last_station_update_times[station_id] = listing_time
                 
                 # station skip lasts until we change station id.
@@ -310,20 +284,24 @@ class ImportPlugin(plugins.ImportPluginBase):
                 supply_units = int(listing['supply'])
                 supply_level = int(listing.get('supply_bracket') or '-1')
                 
-                self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
+                if is_debug:
+                    self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
                 cursor.execute(stmt_add_listing, (
                         station_id, item_id, listing_time, from_live,
                         demand_price, demand_units, demand_level,
                         supply_price, supply_units, supply_level,
                 ))
-            
-        prog.clear()
-        
-        # Do a final commit to be sure
-        db.commit()
-        
-        self.tdenv.NOTE("Optimizing database...")
-        db.execute("VACUUM")
+                transaction_items += 1
+
+        # These will take a little while, which has four steps, so we'll make it a counter.
+        with pbar.Progress(1, 40, prefix="Saving"):
+            # Do a final commit to be sure
+            cursor.execute("COMMIT")
+
+        if self.getOption("optimize"):
+            with pbar.Progress(1, 40, prefix="Optimizing"):
+                db.execute("VACUUM")
+
         self.tdb.close()
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
@@ -349,7 +327,7 @@ class ImportPlugin(plugins.ImportPluginBase):
         
         # We can probably safely assume that the plugin
         # has never been run if the db file doesn't exist.
-        if not (self.tdb.dataPath / Path("TradeDangerous.db")).exists():
+        if not self.tdb.dbPath.exists():
             self.options["clean"] = True
         
         if self.getOption("clean"):
@@ -392,8 +370,11 @@ class ImportPlugin(plugins.ImportPluginBase):
                 if rib_path.exists():
                     rib_path.unlink()
                 ri_path.rename(rib_path)
-            
+
+            self.tdb.close()
+
             self.tdb.reloadCache()
+            self.tdb.close()
             
             # Now it's safe to move RareItems back.
             if ri_path.exists():
@@ -451,7 +432,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             if self.downloadFile(self.upgradesPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlOutfitting, self.FDevOutfittingPath)
                 buildCache = True
-        
+
         if self.getOption("ship"):
             if self.downloadFile(self.shipPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlShipyard, self.FDevShipyardPath)
@@ -486,16 +467,18 @@ class ImportPlugin(plugins.ImportPluginBase):
         if buildCache:
             self.tdb.close()
             self.tdb.reloadCache()
-        
+            self.tdb.close()
+
         if self.getOption("purge"):
             self.purgeSystems()
+            self.tdb.close()
         
         if self.getOption("listings"):
             if self.downloadFile(self.listingsPath) or self.getOption("force"):
                 self.importListings(self.listingsPath)
             if self.downloadFile(self.liveListingsPath) or self.getOption("force"):
                 self.importListings(self.liveListingsPath)
-        
+
         if self.getOption("listings"):
             self.tdenv.NOTE("Regenerating .prices file.")
             cache.regeneratePricesFile(self.tdb, self.tdenv)

tradedangerous/templates/TradeDangerous.sql

@@ -92,10 +92,9 @@ CREATE TABLE Station
    UNIQUE (station_id),
 
    FOREIGN KEY (system_id) REFERENCES System(system_id)
-    ON UPDATE CASCADE
     ON DELETE CASCADE
- );
-CREATE INDEX idx_station_by_system ON Station (system_id, station_id);
+ ) WITHOUT ROWID;
+CREATE INDEX idx_station_by_system ON Station (system_id);
 CREATE INDEX idx_station_by_name ON Station (name);
 
 
@@ -226,11 +225,45 @@ CREATE TABLE StationItem
     ON UPDATE CASCADE ON DELETE CASCADE,
   FOREIGN KEY (item_id) REFERENCES Item(item_id)
     ON UPDATE CASCADE ON DELETE CASCADE
-);
+) WITHOUT ROWID;
 CREATE INDEX si_mod_stn_itm ON StationItem(modified, station_id, item_id);
 CREATE INDEX si_itm_dmdpr ON StationItem(item_id, demand_price) WHERE demand_price > 0;
 CREATE INDEX si_itm_suppr ON StationItem(item_id, supply_price) WHERE supply_price > 0;
 
+-- Not used yet
+CREATE TABLE IF NOT EXISTS StationDemand
+(
+    station_id      INTEGER NOT NULL,
+    item_id         INTEGER NOT NULL,
+    price           INTEGER NOT NULL,
+    units           INTEGER NOT NULL,
+    level           INTEGER NOT NULL,
+    modified        INTEGER NOT NULL,
+    from_live       INTEGER NOT NULL DEFAULT 0,
+    CONSTRAINT pk_StationDemand PRIMARY KEY (station_id, item_id),
+    CONSTRAINT fk_StationDemand_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
+    CONSTRAINT fk_StationDemand_item_id_Item       FOREIGN KEY (item_id)    REFERENCES Item    (item_id)    ON DELETE CASCADE
+) WITHOUT ROWID;
+DELETE FROM StationDemand;
+CREATE INDEX idx_StationDemand_item ON StationDemand (item_id);
+
+-- Not used yet
+CREATE TABLE IF NOT EXISTS StationSupply
+(
+    station_id      INTEGER NOT NULL,
+    item_id         INTEGER NOT NULL,
+    price           INTEGER NOT NULL,
+    units           INTEGER NOT NULL,
+    level           INTEGER NOT NULL,
+    modified        INTEGER NOT NULL,
+    from_live       INTEGER NOT NULL DEFAULT 0,
+    CONSTRAINT pk_StationSupply PRIMARY KEY (station_id, item_id),
+    CONSTRAINT fk_StationSupply_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
+    CONSTRAINT fk_StationSupply_item_id_Item       FOREIGN KEY (item_id)    REFERENCES Item    (item_id)    ON DELETE CASCADE
+) WITHOUT ROWID;
+DELETE FROM StationSupply;
+CREATE INDEX idx_StationSupply_item ON StationSupply (item_id);
+
 CREATE VIEW StationBuying AS
 SELECT  station_id,
         item_id,