tradedangerous 11.0.3__tar.gz → 11.0.5__tar.gz

{tradedangerous-11.0.3/tradedangerous.egg-info → tradedangerous-11.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 11.0.3
+Version: 11.0.5
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/setup.py

@@ -30,7 +30,7 @@ setup(name = package,
         install_requires = ["requests", "appJar", "ijson", "rich"],
         setup_requires = ["pytest-runner"],
         tests_require = ["pytest"],
-        packages = ['tradedangerous', 'tradedangerous.commands', 'tradedangerous.mfd', 'tradedangerous.mfd.saitek', 'tradedangerous.misc', 'tradedangerous.plugins'],
+        packages = ['.', 'tradedangerous', 'tradedangerous.commands', 'tradedangerous.mfd', 'tradedangerous.mfd.saitek', 'tradedangerous.misc', 'tradedangerous.plugins'],
         url = "https://github.com/eyeonus/Trade-Dangerous",
         project_urls = {
             "Bug Tracker": "https://github.com/eyeonus/Trade-Dangerous/issues",
@@ -57,8 +57,8 @@ setup(name = package,
         package_data = {"tradedangerous": ["templates/TradeDangerous.sql", "templates/Added.csv", "templates/Category.csv", "templates/RareItem.csv"]},
         entry_points = {
             "console_scripts": [
-                "trade=tradedangerous.trade:main",
-                "tradegui=tradedangerous.tradegui:main"
+                "trade=trade:main",
+                "tradegui=tradegui:main"
             ]
         },
         zip_safe = False

{tradedangerous-11.0.3/tradedangerous → tradedangerous-11.0.5}/trade.py

@@ -33,19 +33,12 @@
 # DEVELOPERS: If you are a programmer who wants TD to do something
 # cool, please see the TradeDB and TradeCalc modules. TD is designed
 # to empower other programmers to do cool stuff.
-from __future__ import annotations
-
 from tradedangerous import cli
 
-import sys
-    
-
-def main(argv: list[tuple] = None) -> None:
-    """ Entry point for the TradeDangerous command-line app. """
-    if argv is None:
-        argv = sys.argv
-    cli.main(argv)
-
+def main(argv = None):
+  import sys
+  cli.main(sys.argv)
 
 if __name__ == "__main__":
-    cli.main(sys.argv)
+  import sys
+  cli.main(sys.argv)

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/cache.py

@@ -206,7 +206,7 @@ class MultipleItemEntriesError(DuplicateKeyError):
         super().__init__(fromFile, lineNo, 'item', item, prevLineNo)
 
 
-class SyntaxError(BuildCacheBaseException):
+class InvalidLineError(BuildCacheBaseException):
     """
     Raised when an invalid line is read.
     Attributes:
@@ -258,7 +258,7 @@ def parseSupply(pricesFile: Path, lineNo: int, category: str, reading: str) -> t
     #   High <- 'H';
     if reading == '?':
         return -1, -1
-    elif reading == '-':
+    if reading == '-':
         return 0, 0
     
     # extract the left most digits into unit and the last character into the level reading.
@@ -604,13 +604,13 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
         if text.startswith('@'):
             matches = systemStationRe.match(text)
             if not matches:
-                raise SyntaxError(priceFile, lineNo, "Unrecognized '@' line", text)
+                raise InvalidLineError(priceFile, lineNo, "Unrecognized '@' line", text)
             changeStation(matches)
             continue
         
         if not stationID:
             # Need a station to process any other type of line.
-            raise SyntaxError(priceFile, lineNo, "Expecting '@ SYSTEM / Station' line", text)
+            raise InvalidLineError(priceFile, lineNo, "Expecting '@ SYSTEM / Station' line", text)
         if stationID == DELETED:
             # Ignore all values from a deleted station/system.
             continue
@@ -625,7 +625,7 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
         # ## "Item sell buy ..." lines.
         matches = newItemPriceRe.match(text)
         if not matches:
-            raise SyntaxError(priceFile, lineNo, "Unrecognized line/syntax", text)
+            raise InvalidLineError(priceFile, lineNo, "Unrecognized line/syntax", text)
         
         processItemLine(matches)
     
@@ -639,7 +639,7 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
             "if you /need/ to persist them."
         )
     
-    stations = tuple((ID,) for ID in processedStations.keys())
+    stations = tuple((ID,) for ID in processedStations)
     return stations, items, zeros, newItems, updtItems, ignItems, numSys
 
 
@@ -649,9 +649,9 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
 def processPricesFile(tdenv: TradeEnv, db: sqlite3.Connection, pricesPath: Path, pricesFh: Optional[TextIO] = None, defaultZero: bool = False) -> None:
     tdenv.DEBUG0("Processing Prices file '{}'", pricesPath)
     
-    with pricesFh or pricesPath.open('r', encoding='utf-8') as pricesFh:
+    with (pricesFh or pricesPath.open('r', encoding='utf-8')) as fh:
         stations, items, zeros, newItems, updtItems, ignItems, numSys = processPrices(
-            tdenv, pricesFh, db, defaultZero
+            tdenv, fh, db, defaultZero
         )
     
     if not tdenv.mergeImport:
@@ -907,7 +907,7 @@ def processImportFile(tdenv, db, importPath, tableName):
                 try:
                     db.execute(sql_stmt, linein)
                     importCount += 1
-                except Exception as e:
+                except Exception as e:  # pylint: disable=broad-exception-caught
                     tdenv.WARN(
                         "*** INTERNAL ERROR: {err}\n"
                         "CSV File: {file}:{line}\n"

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/cli.py

@@ -32,34 +32,35 @@
 # to empower other programmers to do cool stuff.
 
 import os
+import sys
 import traceback
 
 from . import commands
+from . import tradeexcept
 from .commands import exceptions
 from .plugins import PluginException
 
-
 from . import tradedb
 
+if "CPROF" in os.environ:
+    import cProfile
+
 
 def main(argv = None):
-    import sys
     if not argv:
         argv = sys.argv
-    if sys.hexversion < 0x03040200:
+    if sys.hexversion < 0x03070000:
         raise SystemExit(
-            "Sorry: TradeDangerous requires Python 3.4.2 or higher.\n"
+            "Sorry: TradeDangerous requires Python 3.7 or higher.\n"
             "For assistance, see:\n"
             "\tBug Tracker: https://github.com/eyeonus/Trade-Dangerous/issues\n"
             "\tDocumentation: https://github.com/eyeonus/Trade-Dangerous/wiki\n"
             "\tEDForum Thread: https://forums.frontier.co.uk/showthread.php/441509\n"
         )
-    from . import tradeexcept
     
     try:
         try:
             if "CPROF" in os.environ:
-                import cProfile
                 cProfile.run("trade(argv)")
             else:
                 trade(argv)
@@ -79,7 +80,7 @@ def main(argv = None):
         print()
         print(traceback.format_exc())
         print(
-            "Please report this bug (http://kfs.org/td/issues). You may be "
+            "Please report this bug (http://github.com/eyeonus/Trade-Dangerous/issues). You may be "
             "able to work around it by using the '-q' parameter. Windows "
             "users may be able to use 'chcp.com 65001' to tell the console "
             "you want to support UTF-8 characters."

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/csvexport.py

@@ -36,6 +36,7 @@ def search_keyList(items, val):
     for row in items:
         if row['from'] == row['to'] == val:
             return row
+    return None
 
 def getUniqueIndex(conn, tableName):
     """ return all unique columns """

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/formatting.py

@@ -8,7 +8,9 @@ import itertools
 import typing
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Callable, Optional
+    from typing import Any, Optional
+    from collections.abc import Callable
+
 
 class ColumnFormat:
     """
@@ -197,8 +199,7 @@ if __name__ == '__main__':
     print("Simple usage:")
     present()
     
-    print()
-    print("Adding age ColumnFormat:")
-    
-    rowFmt.append(after='Name', col=ColumnFormat("Age", '>', 3, pre='|', post='|', key=lambda row: row['age']))
-    present()
+    #print()
+    #print("Adding age ColumnFormat:")
+    #rowFmt.append(after='Name', col=ColumnFormat("Age", '>', 3, pre='|', post='|', key=lambda row: row['age']))
+    #present()

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/plugins/eddblink_plug.py

@@ -1,7 +1,8 @@
-# ----------------------------------------------------------------
-# Import plugin that uses data files from EDDB.io and (optionally)
-# a EDDBlink_listener server to update the Database.
-# ----------------------------------------------------------------
+"""
+Import plugin that uses data files from 
+https://elite.tromador.com/ to update the Database.
+"""
+from __future__ import annotations
 import certifi
 import csv
 import datetime
@@ -10,42 +11,48 @@ import os
 import sqlite3
 import ssl
 import time
+import typing
 
 from urllib import request
-from calendar import timegm
 from pathlib import Path
 
 from .. import plugins, cache, transfers
 from ..misc import progress as pbar
 from ..plugins import PluginException
 
+
+if typing.TYPE_CHECKING:
+    from typing import Optional
+    from .. tradeenv import TradeEnv
+
+
 # Constants
 BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
 CONTEXT=ssl.create_default_context(cafile=certifi.where())
 
 
-def request_url(url, headers=None):
+def _request_url(url, headers=None):
     data = None
     if headers:
         data = bytes(json.dumps(headers), encoding="utf-8")
     
-    return request.urlopen(request.Request(url, data=data), context=CONTEXT)
+    return request.urlopen(request.Request(url, data=data), context=CONTEXT, timeout=90)
 
 
 class DecodingError(PluginException):
     pass
 
 
-def file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
+def _file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
     """ counts the number of newline characters in a given file. """
-    # Pre-allocate a buffer so we're not putting pressure on the garbage collector,
-    # capture it's counting method so we don't have to keep looking that up on
-    # large files.
+    # Pre-allocate a buffer so we aren't putting pressure on the garbage collector.
     buf = bytearray(bufsize)
+    
+    # Capture it's counting method, so we don't have to keep looking that up on
+    # large files.
     counter = buf.count
-
+    
     total = 0
-
     with from_file.open("rb") as fh:
         # Capture the 'readinto' method to avoid lookups.
         reader = fh.readinto
@@ -60,10 +67,46 @@ def file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
         # when 0 <= read < bufsize we're on the last page of the
         # file, so we need to take a slice of the buffer, which creates
         # a new object and thus we also have to lookup count. it's trivial
-        # but if you have to do it 10,000x it's definitly not a rounding error.
+        # but if you have to do it 10,000x it's definitely not a rounding error.
         return total + buf[:read].count(b'\n')
 
 
+def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
+    """ Calculates the number of entries in a listing file by counting the lines. """
+    if not listings.exists():
+        tdenv.NOTE("File not found, aborting: {}", listings)
+        return 0
+    
+    tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
+    count = _file_line_count(listings)
+    if count <= 1:
+        if count == 1:
+            tdenv.DEBUG0("Listing count of 1 suggests nothing but a header")
+        else:
+            tdenv.DEBUG0("Listings file is empty, nothing to do.")
+        return 0
+
+    return count + 1  # kfsone: Doesn't the header already make this + 1?
+
+
+def _make_item_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
+    """ helper: retrieve the list of commodities in database. """
+    tdenv.DEBUG0("Getting list of commodities...")
+    return frozenset(cols[0] for cols in db.execute("SELECT item_id FROM Item"))
+
+
+def _make_station_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
+    """ helper: retrieve the list of station IDs in database. """
+    tdenv.DEBUG0("Getting list of stations...")
+    return frozenset(cols[0] for cols in db.execute("SELECT station_id FROM Station"))
+
+
+def _collect_station_modified_times(tdenv: TradeEnv, db: sqlite3.Cursor) -> dict[int, int]:
+    """ helper: build a list of the last modified time for all stations by id. """
+    tdenv.DEBUG0("Getting last-update times for stations...")
+    return dict(db.execute("SELECT station_id, strftime('%s', MIN(modified)) FROM StationItem GROUP BY station_id"))
+
+
 class ImportPlugin(plugins.ImportPluginBase):
     """
     Plugin that downloads data from eddb.
@@ -113,61 +156,25 @@ class ImportPlugin(plugins.ImportPluginBase):
     def now(self):
         return datetime.datetime.now()
     
-    def execute(self, sql_cmd, args = None):
-        cur = self.tdb.getDB().cursor()
-        
-        self.tdenv.DEBUG2(f"SQL-Statement:\n'{sql_cmd},{args}'")
-        success = False
-        result = None
-        while not success:
-            try:
-                if args:
-                    result = cur.execute(sql_cmd, args)
-                else:
-                    result = cur.execute(sql_cmd)
-                success = True
-            except sqlite3.OperationalError as e:
-                if "locked" not in str(e):
-                    success = True
-                    raise sqlite3.OperationalError(e)
-                else:
-                    print("(execute) Database is locked, waiting for access.", end = "\r")
-                    time.sleep(1)
-        return result
-    
-    @staticmethod
-    def fetchIter(cursor, arraysize = 1000):
-        """
-        An iterator that uses fetchmany to keep memory usage down
-        and speed up the time to retrieve the results dramatically.
-        """
-        while True:
-            results = cursor.fetchmany(arraysize)
-            if not results:
-                break
-            for result in results:
-                yield result
-    
-
     def downloadFile(self, path):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
         """
         
         def openURL(url):
-            return request_url(url, headers = {'User-Agent': 'Trade-Dangerous'})
+            return _request_url(url, headers = {'User-Agent': 'Trade-Dangerous'})
         
-        if path != self.liveListingsPath and path != self.listingsPath:
-            localPath = self.tdb.dataPath / path
+        if path not in (self.liveListingsPath, self.listingsPath):
+            localPath = Path(self.tdb.dataPath, path)
         else:
-            localPath = self.dataPath / path
+            localPath = Path(self.dataPath, path)
         
         url  = BASE_URL + str(path)
         
         self.tdenv.NOTE("Checking for update to '{}'.", path)
         try:
             response = openURL(url)
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-exception-caught
             self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", BASE_URL + str(path), str(e))
             return False
         
@@ -190,148 +197,137 @@ class ImportPlugin(plugins.ImportPluginBase):
         Purges systems from the System table that do not have any stations claiming to be in them.
         Keeps table from becoming too large because of fleet carriers moving to unpopulated systems.
         """
-        
+        db = self.tdb.getDB()
         self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now())
-        
-        self.execute("PRAGMA foreign_keys = OFF")
-        
-        print("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("DROP TABLE IF EXISTS System_copy")
-        self.execute("""CREATE TABLE System_copy AS SELECT * FROM System
+
+        db.execute("PRAGMA foreign_keys = OFF")
+
+        self.tdenv.DEBUG0("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("DROP TABLE IF EXISTS System_copy")
+        db.execute("""CREATE TABLE System_copy AS SELECT * FROM System
                             WHERE system_id IN (SELECT system_id FROM Station)
                     """)
-        
-        print("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("DELETE FROM System")
-        self.execute("INSERT INTO System SELECT * FROM System_copy")
-        
-        print("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("PRAGMA foreign_keys = ON")
-        self.execute("DROP TABLE IF EXISTS System_copy")
-        
+
+        self.tdenv.DEBUG0("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("DELETE FROM System")
+        db.execute("INSERT INTO System SELECT * FROM System_copy")
+
+        self.tdenv.DEBUG0("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("PRAGMA foreign_keys = ON")
+        db.execute("DROP TABLE IF EXISTS System_copy")
+
+        db.commit()
+
         self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now())
     
-    def commit(self):
-        success = False
-        while not success:
-            try:
-                self.tdb.getDB().commit()
-                success = True
-            except sqlite3.OperationalError:
-                print("(commit) Database is locked, waiting for access.", end = "\r")
-                time.sleep(1)
-    
     def importListings(self, listings_file):
         """
-        Updates the market data (AKA the StationItem table) using listings.csv
+        Updates the market data (AKA the StationItem table) using listings_file
         Writes directly to database.
         """
+        listings_path = Path(self.dataPath, listings_file).absolute()
+        from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
+        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
         
-        self.tdenv.NOTE("Processing market data from {}: Start time = {}", listings_file, self.now())
-        if not (self.dataPath / listings_file).exists():
-            self.tdenv.NOTE("File not found, aborting: {}", (self.dataPath / listings_file))
+        total = _count_listing_entries(self.tdenv, listings_path)
+        if not total:
             return
+
+        stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
+        stmt_flush_station   = """DELETE from StationItem WHERE station_id = ?"""
+        stmt_add_listing     = """
+            INSERT OR IGNORE INTO StationItem (
+                station_id, item_id, modified, from_live,
+                demand_price, demand_units, demand_level,
+                supply_price, supply_units, supply_level
+            )
+            VALUES (
+                ?, ?, datetime(?, 'unixepoch'), ?,
+                ?, ?, ?,
+                ?, ?, ?
+            )
+        """
         
-        total = 1
-        
-        from_live = 0 if listings_file == self.listingsPath else 1
-        
-        self.tdenv.DEBUG0(f"Getting total number of entries in {listings_file}...")
-        listings_path = Path(self.dataPath, listings_file)
-        total += file_line_count(listings_path)
-        
-        liveStmt = """UPDATE StationItem
-                    SET from_live = 0
-                    WHERE station_id = ?"""
-        
-        delStmt = "DELETE from StationItem WHERE station_id = ?"
-        
-        listingStmt = """INSERT OR IGNORE INTO StationItem
-                        (station_id, item_id, modified,
-                         demand_price, demand_units, demand_level,
-                         supply_price, supply_units, supply_level, from_live)
-                        VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"""
-        
-        self.tdenv.DEBUG0("Getting list of commodities...")
-        items = [cols[0] for cols in self.execute("SELECT item_id FROM Item ORDER BY item_id")]
-        
-        self.tdenv.DEBUG0("Getting list of stations...")
-        stationList = {
-            stationID
-            for (stationID,) in self.execute("SELECT station_id FROM Station")
-        }
-        
-        stationItems = dict(self.execute('SELECT station_id, UNIXEPOCH(modified) FROM StationItem').fetchall())
+        # Fetch all the items IDS
+        db = self.tdb.getDB()
+        item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
+        station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
+        last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
         
+        cur_station = None
         self.tdenv.DEBUG0("Processing entries...")
         with listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
             prog = pbar.Progress(total, 50)
-            listings = csv.DictReader(fh)
-            
-            cur_station = -1
+
+            cursor: Optional[sqlite3.Cursor] = db.cursor()
             
-            for listing in listings:
-                if prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}"):
-                    # Do a commit and close the DB every 2%.
-                    # This ensures the listings are put in the DB and the WAL is cleared.
-                    self.commit()
-                    self.tdb.close()
+            for listing in csv.DictReader(fh):
+                prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}")
                 
                 station_id = int(listing['station_id'])
-                if station_id not in stationList:
+                if station_id not in station_lookup:
                     continue
                 
+                listing_time = int(listing['collected_at'])
+                
                 if station_id != cur_station:
-                    cur_station = station_id
-                    skipStation = False
+                    # commit anything from the previous station, get a new cursor
+                    db.commit()
+                    cur_station, skip_station, cursor = station_id, False, db.cursor()
                     
                     # Check if listing already exists in DB and needs updated.
-                    if stationItems.get(station_id):
+                    last_modified: int = int(last_station_update_times.get(station_id, 0))
+                    if last_modified:
                         # When the listings.csv data matches the database, update to make from_live == 0.
-                        if int(listing['collected_at']) == stationItems.get(station_id) and not from_live:
-                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live'.")
-                            self.execute(liveStmt, (cur_station,))
+                        if listing_time == last_modified and not from_live:
+                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
+                            cursor.execute(stmt_unliven_station, (cur_station,))
+                            skip_station = True
+                            continue
+
                         # Unless the import file data is newer, nothing else needs to be done for this station,
                         # so the rest of the listings for this station can be skipped.
-                        if int(listing['collected_at']) <= stationItems.get(station_id):
-                            skipStation = True
+                        if listing_time <= last_modified:
+                            skip_station = True
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
-                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station}.")
-                        self.execute(delStmt, (cur_station,))
-                        # We've deleted all the items from this station, so remove it.
-                        del stationItems[station_id]
-
+                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
+                        cursor.execute(stmt_flush_station, (cur_station,))
+                        last_station_update_times[station_id] = listing_time
                 
-                if skipStation:
+                # station skip lasts until we change station id.
+                if skip_station:
                     continue
                 
                 # Since this station is not being skipped, get the data and prepare for insertion into the DB.
                 item_id = int(listing['commodity_id'])
                 # listings.csv includes rare items, which we are ignoring.
-                if item_id not in items:
+                if item_id not in item_lookup:
                     continue
-                modified = datetime.datetime.utcfromtimestamp(int(listing['collected_at'])).strftime('%Y-%m-%d %H:%M:%S')
+
                 demand_price = int(listing['sell_price'])
                 demand_units = int(listing['demand'])
-                demand_level = int(listing['demand_bracket']) if listing['demand_bracket'] != '' else -1
+                demand_level = int(listing.get('demand_bracket') or '-1')
                 supply_price = int(listing['buy_price'])
                 supply_units = int(listing['supply'])
-                supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1
+                supply_level = int(listing.get('supply_bracket') or '-1')
                 
                 self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
-                self.execute(listingStmt, (station_id, item_id, modified,
-                                    demand_price, demand_units, demand_level,
-                                    supply_price, supply_units, supply_level, from_live))
+                cursor.execute(stmt_add_listing, (
+                        station_id, item_id, listing_time, from_live,
+                        demand_price, demand_units, demand_level,
+                        supply_price, supply_units, supply_level,
+                ))
             
-            # Do a final commit to be sure
-            self.commit()
-            self.tdb.close()
-            
-            while prog.value < prog.maxValue:
-                prog.increment(1, postfix = lambda value, total: " " + str(round(value / total * 100)) + "%")
-            prog.clear()
+        prog.clear()
+        
+        # Do a final commit to be sure
+        db.commit()
+        
+        self.tdenv.NOTE("Optimizing database...")
+        db.execute("VACUUM")
+        self.tdb.close()
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
     
@@ -347,14 +343,13 @@ class ImportPlugin(plugins.ImportPluginBase):
         # have been passed, enable 'listings'.
         default = True
         for option in self.options:
-            # if not option in ('force', 'fallback', 'skipvend', 'progbar'):
-            if option not in ('force', 'skipvend'):
+            if option not in ('force', 'skipvend', 'purge'):
                 default = False
         if default:
             self.options["listings"] = True
         
-        # We can probably safely assume that the plugin has never been run if
-        # the prices file doesn't exist, since the plugin always generates it.
+        # We can probably safely assume that the plugin
+        # has never been run if the db file doesn't exist.
         if not (self.tdb.dataPath / Path("TradeDangerous.db")).exists():
             self.options["clean"] = True
         
@@ -497,7 +492,6 @@ class ImportPlugin(plugins.ImportPluginBase):
         
         if self.getOption("purge"):
             self.purgeSystems()
-            # self.commit()
         
         if self.getOption("listings"):
             if self.downloadFile(self.listingsPath) or self.getOption("force"):

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/plugins/spansh_plug.py

@@ -1,26 +1,29 @@
+""" Plugin for importing data from spansh """
 from __future__ import annotations
 
+from collections import namedtuple
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from pathlib import Path
+from rich.progress import Progress
+
+from .. import plugins, cache, transfers, csvexport, corrections
 
+import sqlite3
 import sys
 import time
 import typing
-from collections import namedtuple
+
+import ijson
+
 if sys.version_info.major == 3 and sys.version_info.minor >= 10:
     from dataclasses import dataclass
 else:
     dataclass = False  # pylint: disable=invalid-name
 
-from rich.progress import Progress
-import ijson
-import sqlite3
-
-from .. import plugins, cache, transfers, csvexport, corrections
-
 if typing.TYPE_CHECKING:
-    from typing import Any, Iterable, Optional
+    from typing import Any, Optional
+    from collections.abc import Iterable
     from .. tradeenv import TradeEnv
 
 SOURCE_URL = 'https://downloads.spansh.co.uk/galaxy_stations.json'
@@ -237,11 +240,11 @@ class ImportPlugin(plugins.ImportPluginBase):
     def commit(self, *, force: bool = False) -> None:
         """ Perform a commit if required, but try not to do a crazy amount of committing. """
         if not force and not self.need_commit:
-            return self.cursor
+            return
         
         if not force and self.commit_limit > 0:
             self.commit_limit -= 1
-            return self.cursor
+            return
         
         db = self.tdb.getDB()
         db.commit()
@@ -291,19 +294,14 @@ class ImportPlugin(plugins.ImportPluginBase):
                     
                     for station, commodities in stations:
                         fq_station_name = f'@{upper_sys}/{station.name}'
-                        if age_cutoff and (now - station.modified) > age_cutoff:
-                            if self.tdenv.detail:
-                                self.print(f'        |  {fq_station_name:50s}  |  Skipping station due to age: {now - station.modified}, ts: {station.modified}')
-                            progress.bump(sys_task)
-                            continue
                         
                         station_info = self.known_stations.get(station.id)
-                        if not station_info:
+                        if not station_info or station.modified > station_info[2]:
                             self.ensure_station(station)
                         elif station_info[1] != station.system_id:
                             self.print(f'        |  {station.name:50s}  |  Megaship station moved, updating system')
                             self.execute("UPDATE Station SET system_id = ? WHERE station_id = ?", station.system_id, station.id, commitable=True)
-                            self.known_stations[station.id] = (station.name, station.system_id)
+                            self.known_stations[station.id] = (station.name, station.system_id, station.modified)
                         
                         items = []
                         db_times = dict(self.execute("SELECT item_id, modified FROM StationItem WHERE station_id = ?", station.id))
@@ -312,12 +310,19 @@ class ImportPlugin(plugins.ImportPluginBase):
                             if commodity.id not in self.known_commodities:
                                 commodity = self.ensure_commodity(commodity)
                             
+                            # We're concerned with the market age, not the station age,
+                            # as they each have their own 'modified' times.
+                            if age_cutoff and (now - commodity.modified) > age_cutoff:
+                                if self.tdenv.detail:
+                                    self.print(f'        |  {fq_station_name:50s}  |  Skipping station due to age: {now - station.modified}, ts: {station.modified}')
+                                break
+                            
                             db_modified = db_times.get(commodity.id)
                             modified = parse_ts(db_modified) if db_modified else None
                             if modified and commodity.modified <= modified:
                                 # All commodities in a station will have the same modified time,
                                 # so no need to check the rest if the fist is older.
-                                if self.tdenv.detail:
+                                if self.tdenv.detail > 2:
                                     self.print(f'        |  {fq_station_name:50s}  |  Skipping older commodity data')
                                 break
                             items.append((station.id, commodity.id, commodity.modified,
@@ -334,8 +339,8 @@ class ImportPlugin(plugins.ImportPluginBase):
                                 ?, ?, ?, ?
                             )""", items, commitable=True)
                             commodity_count += len(items)
-                            # Good time to save data and try to keep the transaction small
-                            self.commit()
+                        # Good time to save data and try to keep the transaction small
+                        self.commit()
                         
                         if commodity_count:
                             station_count += 1
@@ -357,12 +362,6 @@ class ImportPlugin(plugins.ImportPluginBase):
                         progress.update(f"{sys_desc}{DIM} ({total_station_count}:station:, {avg_stations:.1f}per:glowing_star:){CLOSE}")
             
             self.commit()
-            
-            # Need to make sure cached tables are updated, if changes were made
-            # if self.update_cache:
-            #     for table in [ "Item", "Station", "System" ]:
-            #         _, path = csvexport.exportTableToFile( self.tdb, self.tdenv, table )
-            
             self.tdb.close()
             
             # Need to make sure cached tables are updated
@@ -392,7 +391,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             categories.setdefault(commodity.category, []).append(commodity)
         return categories
     
-    def execute(self, query: str, *params, commitable: bool = False) -> Optional[sqlite3.Cursor]:
+    def execute(self, query: str, *params, commitable: bool = False) -> sqlite3.Cursor:
         """ helper method that performs retriable queries and marks the transaction as needing to commit
             if the query is commitable."""
         if commitable:
@@ -404,14 +403,14 @@ class ImportPlugin(plugins.ImportPluginBase):
             except sqlite3.OperationalError as ex:
                 if "no transaction is active" in str(ex):
                     self.print(f"no transaction for {query}")
-                    return
+                    raise
                 if not attempts:
                     raise
                 attempts -= 1
                 self.print(f'Retrying query \'{query}\': {ex!s}')
                 time.sleep(1)
     
-    def executemany(self, query: str, data: Iterable[Any], *, commitable: bool = False) -> Optional[sqlite3.Cursor]:
+    def executemany(self, query: str, data: Iterable[Any], *, commitable: bool = False) -> sqlite3.Cursor:
         """ helper method that performs retriable queries and marks the transaction as needing to commit
             if the query is commitable."""
         if commitable:
@@ -423,7 +422,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             except sqlite3.OperationalError as ex:
                 if "no transaction is active" in str(ex):
                     self.print(f"no transaction for {query}")
-                    return
+                    raise
                 if not attempts:
                     raise
                 attempts -= 1
@@ -442,7 +441,7 @@ class ImportPlugin(plugins.ImportPluginBase):
     def load_known_stations(self) -> dict[int, tuple[str, int]]:
         """ Returns a dictionary of {station_id -> (station_name, system_id)} for all current stations in the database. """
         try:
-            return {cols[0]: (cols[1], cols[2]) for cols in self.cursor.execute('SELECT station_id, name, system_id FROM Station')}
+            return {cols[0]: (cols[1], cols[2], parse_ts(cols[3])) for cols in self.cursor.execute('SELECT station_id, name, system_id, modified FROM Station')}
         except Exception as e:  # pylint: disable=broad-except
             self.print("[purple]:thinking_face:Assuming no station data yet")
             self.tdenv.DEBUG0(f"load_known_stations query raised {e}")
@@ -474,7 +473,7 @@ class ImportPlugin(plugins.ImportPluginBase):
         """ Adds a record for a station, and registers the station in the known_stations dict. """
         self.execute(
             '''
-            INSERT INTO Station (
+            INSERT OR REPLACE INTO Station (
                 system_id, station_id, name,
                 ls_from_star, max_pad_size,
                 market, blackmarket, shipyard, outfitting,
@@ -510,9 +509,10 @@ class ImportPlugin(plugins.ImportPluginBase):
             station.type,
             commitable=True,
         )
+        note = "Updated" if self.known_stations.get(station.id) else "Added"
         if self.tdenv.detail > 1:
-            self.print(f'        |  {station.name:50s}  |  Added missing station')
-        self.known_stations[station.id] = (station.name, station.system_id)
+            self.print(f'        |  {station.name:50s}  |  {note} station')
+        self.known_stations[station.id] = (station.name, station.system_id, station.modified)
     
     def ensure_commodity(self, commodity: Commodity):
         """ Adds a record for a commodity and registers the commodity in the known_commodities dict. """

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '11.0.3'
+__version__ = '11.0.5'
 

{tradedangerous-11.0.3 → tradedangerous-11.0.5/tradedangerous.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 11.0.3
+Version: 11.0.5
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous.egg-info/SOURCES.txt

@@ -3,6 +3,8 @@ README.md
 pyproject.toml
 setup.cfg
 setup.py
+trade.py
+tradegui.py
 tests/test_bootstrap_commands.py
 tests/test_bootstrap_plugins.py
 tests/test_cache.py
@@ -28,12 +30,10 @@ tradedangerous/mapping.py
 tradedangerous/prices.py
 tradedangerous/submit-distances.py
 tradedangerous/tools.py
-tradedangerous/trade.py
 tradedangerous/tradecalc.py
 tradedangerous/tradedb.py
 tradedangerous/tradeenv.py
 tradedangerous/tradeexcept.py
-tradedangerous/tradegui.py
 tradedangerous/transfers.py
 tradedangerous/utils.py
 tradedangerous/version.py

tradedangerous-11.0.5/tradedangerous.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+trade = trade:main
+tradegui = tradegui:main

{tradedangerous-11.0.3 → tradedangerous-11.0.5}/tradedangerous.egg-info/top_level.txt

@@ -1 +1,2 @@
+
 tradedangerous

tradedangerous-11.0.3/tradedangerous.egg-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-trade = tradedangerous.trade:main
-tradegui = tradedangerous.tradegui:main