tradedangerous 11.0.2__py3-none-any.whl → 11.0.4__py3-none-any.whl

tradedangerous/cache.py

@@ -206,7 +206,7 @@ class MultipleItemEntriesError(DuplicateKeyError):
         super().__init__(fromFile, lineNo, 'item', item, prevLineNo)
 
 
-class SyntaxError(BuildCacheBaseException):
+class InvalidLineError(BuildCacheBaseException):
     """
     Raised when an invalid line is read.
     Attributes:
@@ -258,7 +258,7 @@ def parseSupply(pricesFile: Path, lineNo: int, category: str, reading: str) -> t
     #   High <- 'H';
     if reading == '?':
         return -1, -1
-    elif reading == '-':
+    if reading == '-':
         return 0, 0
     
     # extract the left most digits into unit and the last character into the level reading.
@@ -604,13 +604,13 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
         if text.startswith('@'):
             matches = systemStationRe.match(text)
             if not matches:
-                raise SyntaxError(priceFile, lineNo, "Unrecognized '@' line", text)
+                raise InvalidLineError(priceFile, lineNo, "Unrecognized '@' line", text)
             changeStation(matches)
             continue
         
         if not stationID:
             # Need a station to process any other type of line.
-            raise SyntaxError(priceFile, lineNo, "Expecting '@ SYSTEM / Station' line", text)
+            raise InvalidLineError(priceFile, lineNo, "Expecting '@ SYSTEM / Station' line", text)
         if stationID == DELETED:
             # Ignore all values from a deleted station/system.
             continue
@@ -625,7 +625,7 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
         # ## "Item sell buy ..." lines.
         matches = newItemPriceRe.match(text)
         if not matches:
-            raise SyntaxError(priceFile, lineNo, "Unrecognized line/syntax", text)
+            raise InvalidLineError(priceFile, lineNo, "Unrecognized line/syntax", text)
         
         processItemLine(matches)
     
@@ -639,7 +639,7 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
             "if you /need/ to persist them."
         )
     
-    stations = tuple((ID,) for ID in processedStations.keys())
+    stations = tuple((ID,) for ID in processedStations)
     return stations, items, zeros, newItems, updtItems, ignItems, numSys
 
 
@@ -649,9 +649,9 @@ def processPrices(tdenv: TradeEnv, priceFile: Path, db: sqlite3.Connection, defa
 def processPricesFile(tdenv: TradeEnv, db: sqlite3.Connection, pricesPath: Path, pricesFh: Optional[TextIO] = None, defaultZero: bool = False) -> None:
     tdenv.DEBUG0("Processing Prices file '{}'", pricesPath)
     
-    with pricesFh or pricesPath.open('r', encoding='utf-8') as pricesFh:
+    with (pricesFh or pricesPath.open('r', encoding='utf-8')) as fh:
         stations, items, zeros, newItems, updtItems, ignItems, numSys = processPrices(
-            tdenv, pricesFh, db, defaultZero
+            tdenv, fh, db, defaultZero
         )
     
     if not tdenv.mergeImport:
@@ -907,7 +907,7 @@ def processImportFile(tdenv, db, importPath, tableName):
                 try:
                     db.execute(sql_stmt, linein)
                     importCount += 1
-                except Exception as e:
+                except Exception as e:  # pylint: disable=broad-exception-caught
                     tdenv.WARN(
                         "*** INTERNAL ERROR: {err}\n"
                         "CSV File: {file}:{line}\n"

tradedangerous/cli.py

@@ -32,34 +32,35 @@
 # to empower other programmers to do cool stuff.
 
 import os
+import sys
 import traceback
 
 from . import commands
+from . import tradeexcept
 from .commands import exceptions
 from .plugins import PluginException
 
-
 from . import tradedb
 
+if "CPROF" in os.environ:
+    import cProfile
+
 
 def main(argv = None):
-    import sys
     if not argv:
         argv = sys.argv
-    if sys.hexversion < 0x03040200:
+    if sys.hexversion < 0x03070000:
         raise SystemExit(
-            "Sorry: TradeDangerous requires Python 3.4.2 or higher.\n"
+            "Sorry: TradeDangerous requires Python 3.7 or higher.\n"
             "For assistance, see:\n"
             "\tBug Tracker: https://github.com/eyeonus/Trade-Dangerous/issues\n"
             "\tDocumentation: https://github.com/eyeonus/Trade-Dangerous/wiki\n"
             "\tEDForum Thread: https://forums.frontier.co.uk/showthread.php/441509\n"
         )
-    from . import tradeexcept
     
     try:
         try:
             if "CPROF" in os.environ:
-                import cProfile
                 cProfile.run("trade(argv)")
             else:
                 trade(argv)
@@ -79,7 +80,7 @@ def main(argv = None):
         print()
         print(traceback.format_exc())
         print(
-            "Please report this bug (http://kfs.org/td/issues). You may be "
+            "Please report this bug (http://github.com/eyeonus/Trade-Dangerous/issues). You may be "
             "able to work around it by using the '-q' parameter. Windows "
             "users may be able to use 'chcp.com 65001' to tell the console "
             "you want to support UTF-8 characters."

tradedangerous/commands/parsing.py

@@ -60,7 +60,7 @@ class PadSizeArgument(int):
             'dest': 'padSize',
             'metavar': 'PADSIZES',
             'type': 'padsize',
-            'choices': 'SML?',
+            'choices': 'SsMmLl?',
         }
 
 
@@ -153,7 +153,7 @@ class PlanetaryArgument(int):
             'dest': 'planetary',
             'metavar': 'PLANETARY',
             'type': 'planetary',
-            'choices': 'YN??',
+            'choices': 'YyNn?',
         }
 
 
@@ -181,7 +181,7 @@ class FleetCarrierArgument(int):
             'dest': 'fleet',
             'metavar': 'FLEET',
             'type': 'fleet',
-            'choices': 'YN?',
+            'choices': 'YyNn?',
         }
 
 class OdysseyArgument(int):
@@ -208,7 +208,7 @@ class OdysseyArgument(int):
             'dest': 'odyssey',
             'metavar': 'ODYSSEY',
             'type': 'odyssey',
-            'choices': 'YN?',
+            'choices': 'YyNn?',
         }
 
 

tradedangerous/csvexport.py

@@ -36,6 +36,7 @@ def search_keyList(items, val):
     for row in items:
         if row['from'] == row['to'] == val:
             return row
+    return None
 
 def getUniqueIndex(conn, tableName):
     """ return all unique columns """

tradedangerous/formatting.py

@@ -8,7 +8,9 @@ import itertools
 import typing
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Callable, Optional
+    from typing import Any, Optional
+    from collections.abc import Callable
+
 
 class ColumnFormat:
     """
@@ -197,8 +199,7 @@ if __name__ == '__main__':
     print("Simple usage:")
     present()
     
-    print()
-    print("Adding age ColumnFormat:")
-    
-    rowFmt.append(after='Name', col=ColumnFormat("Age", '>', 3, pre='|', post='|', key=lambda row: row['age']))
-    present()
+    #print()
+    #print("Adding age ColumnFormat:")
+    #rowFmt.append(after='Name', col=ColumnFormat("Age", '>', 3, pre='|', post='|', key=lambda row: row['age']))
+    #present()

tradedangerous/misc/edsc.py

@@ -4,19 +4,7 @@ from collections import namedtuple
 from urllib.request import Request, urlopen
 
 import json
-
-try:
-    import requests
-except ImportError as e:
-    import pip
-    print("ERROR: Unable to load the Python 'requests' package.")
-    approval = input(
-        "Do you want me to try and install it with the package manager (y/n)? "
-    )
-    if approval.lower() != 'y':
-        raise e
-    pip.main(["install", "--upgrade", "requests"])
-    import requests
+import requests
 
 
 def edsc_log(apiCall, params, jsonData=None, error=None):

tradedangerous/misc/edsm.py

@@ -6,19 +6,7 @@ uses EDSM - https://www.edsm.net/api
 """
 
 import json
-
-try:
-    import requests
-except ImportError as e:
-    import pip
-    print("ERROR: Unable to load the Python 'requests' package.")
-    approval = input(
-        "Do you want me to try and install it with the package manager (y/n)? "
-    )
-    if approval.lower() != 'y':
-        raise e
-    pip.main(["install", "--upgrade", "requests"])
-    import requests
+import requests
 
 
 def edsm_log(apiCall, url, params, jsonData=None, error=None):

tradedangerous/plugins/eddblink_plug.py

@@ -1,7 +1,8 @@
-# ----------------------------------------------------------------
-# Import plugin that uses data files from EDDB.io and (optionally)
-# a EDDBlink_listener server to update the Database.
-# ----------------------------------------------------------------
+"""
+Import plugin that uses data files from 
+https://elite.tromador.com/ to update the Database.
+"""
+from __future__ import annotations
 import certifi
 import csv
 import datetime
@@ -10,42 +11,48 @@ import os
 import sqlite3
 import ssl
 import time
+import typing
 
 from urllib import request
-from calendar import timegm
 from pathlib import Path
 
 from .. import plugins, cache, transfers
 from ..misc import progress as pbar
 from ..plugins import PluginException
 
+
+if typing.TYPE_CHECKING:
+    from typing import Optional
+    from .. tradeenv import TradeEnv
+
+
 # Constants
 BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
 CONTEXT=ssl.create_default_context(cafile=certifi.where())
 
 
-def request_url(url, headers=None):
+def _request_url(url, headers=None):
     data = None
     if headers:
         data = bytes(json.dumps(headers), encoding="utf-8")
     
-    return request.urlopen(request.Request(url, data=data), context=CONTEXT)
+    return request.urlopen(request.Request(url, data=data), context=CONTEXT, timeout=90)
 
 
 class DecodingError(PluginException):
     pass
 
 
-def file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
+def _file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
     """ counts the number of newline characters in a given file. """
-    # Pre-allocate a buffer so we're not putting pressure on the garbage collector,
-    # capture it's counting method so we don't have to keep looking that up on
-    # large files.
+    # Pre-allocate a buffer so we aren't putting pressure on the garbage collector.
     buf = bytearray(bufsize)
+    
+    # Capture it's counting method, so we don't have to keep looking that up on
+    # large files.
     counter = buf.count
-
+    
     total = 0
-
     with from_file.open("rb") as fh:
         # Capture the 'readinto' method to avoid lookups.
         reader = fh.readinto
@@ -60,10 +67,46 @@ def file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
         # when 0 <= read < bufsize we're on the last page of the
         # file, so we need to take a slice of the buffer, which creates
         # a new object and thus we also have to lookup count. it's trivial
-        # but if you have to do it 10,000x it's definitly not a rounding error.
+        # but if you have to do it 10,000x it's definitely not a rounding error.
         return total + buf[:read].count(b'\n')
 
 
+def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
+    """ Calculates the number of entries in a listing file by counting the lines. """
+    if not listings.exists():
+        tdenv.NOTE("File not found, aborting: {}", listings)
+        return 0
+    
+    tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
+    count = _file_line_count(listings)
+    if count <= 1:
+        if count == 1:
+            tdenv.DEBUG0("Listing count of 1 suggests nothing but a header")
+        else:
+            tdenv.DEBUG0("Listings file is empty, nothing to do.")
+        return 0
+
+    return count + 1  # kfsone: Doesn't the header already make this + 1?
+
+
+def _make_item_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
+    """ helper: retrieve the list of commodities in database. """
+    tdenv.DEBUG0("Getting list of commodities...")
+    return frozenset(cols[0] for cols in db.execute("SELECT item_id FROM Item"))
+
+
+def _make_station_id_lookup(tdenv: TradeEnv, db: sqlite3.Cursor) -> frozenset[int]:
+    """ helper: retrieve the list of station IDs in database. """
+    tdenv.DEBUG0("Getting list of stations...")
+    return frozenset(cols[0] for cols in db.execute("SELECT station_id FROM Station"))
+
+
+def _collect_station_modified_times(tdenv: TradeEnv, db: sqlite3.Cursor) -> dict[int, int]:
+    """ helper: build a list of the last modified time for all stations by id. """
+    tdenv.DEBUG0("Getting last-update times for stations...")
+    return dict(db.execute("SELECT station_id, strftime('%s', MIN(modified)) FROM StationItem GROUP BY station_id"))
+
+
 class ImportPlugin(plugins.ImportPluginBase):
     """
     Plugin that downloads data from eddb.
@@ -113,61 +156,25 @@ class ImportPlugin(plugins.ImportPluginBase):
     def now(self):
         return datetime.datetime.now()
     
-    def execute(self, sql_cmd, args = None):
-        cur = self.tdb.getDB().cursor()
-        
-        self.tdenv.DEBUG2(f"SQL-Statement:\n'{sql_cmd},{args}'")
-        success = False
-        result = None
-        while not success:
-            try:
-                if args:
-                    result = cur.execute(sql_cmd, args)
-                else:
-                    result = cur.execute(sql_cmd)
-                success = True
-            except sqlite3.OperationalError as e:
-                if "locked" not in str(e):
-                    success = True
-                    raise sqlite3.OperationalError(e)
-                else:
-                    print("(execute) Database is locked, waiting for access.", end = "\r")
-                    time.sleep(1)
-        return result
-    
-    @staticmethod
-    def fetchIter(cursor, arraysize = 1000):
-        """
-        An iterator that uses fetchmany to keep memory usage down
-        and speed up the time to retrieve the results dramatically.
-        """
-        while True:
-            results = cursor.fetchmany(arraysize)
-            if not results:
-                break
-            for result in results:
-                yield result
-    
-
     def downloadFile(self, path):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
         """
         
         def openURL(url):
-            return request_url(url, headers = {'User-Agent': 'Trade-Dangerous'})
+            return _request_url(url, headers = {'User-Agent': 'Trade-Dangerous'})
         
-        if path != self.liveListingsPath and path != self.listingsPath:
-            localPath = self.tdb.dataPath / path
+        if path not in (self.liveListingsPath, self.listingsPath):
+            localPath = Path(self.tdb.dataPath, path)
         else:
-            localPath = self.dataPath / path
+            localPath = Path(self.dataPath, path)
         
         url  = BASE_URL + str(path)
         
         self.tdenv.NOTE("Checking for update to '{}'.", path)
         try:
             response = openURL(url)
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-exception-caught
             self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", BASE_URL + str(path), str(e))
             return False
         
@@ -190,148 +197,137 @@ class ImportPlugin(plugins.ImportPluginBase):
         Purges systems from the System table that do not have any stations claiming to be in them.
         Keeps table from becoming too large because of fleet carriers moving to unpopulated systems.
         """
-        
+        db = self.tdb.getDB()
         self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now())
-        
-        self.execute("PRAGMA foreign_keys = OFF")
-        
-        print("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("DROP TABLE IF EXISTS System_copy")
-        self.execute("""CREATE TABLE System_copy AS SELECT * FROM System
+
+        db.execute("PRAGMA foreign_keys = OFF")
+
+        self.tdenv.DEBUG0("Saving systems with stations.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("DROP TABLE IF EXISTS System_copy")
+        db.execute("""CREATE TABLE System_copy AS SELECT * FROM System
                             WHERE system_id IN (SELECT system_id FROM Station)
                     """)
-        
-        print("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("DELETE FROM System")
-        self.execute("INSERT INTO System SELECT * FROM System_copy")
-        
-        print("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r")
-        self.execute("PRAGMA foreign_keys = ON")
-        self.execute("DROP TABLE IF EXISTS System_copy")
-        
+
+        self.tdenv.DEBUG0("Erasing table and reinserting kept systems.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("DELETE FROM System")
+        db.execute("INSERT INTO System SELECT * FROM System_copy")
+
+        self.tdenv.DEBUG0("Removing copy.... " + str(self.now()) + "\t\t\t\t", end="\r")
+        db.execute("PRAGMA foreign_keys = ON")
+        db.execute("DROP TABLE IF EXISTS System_copy")
+
+        db.commit()
+
         self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now())
     
-    def commit(self):
-        success = False
-        while not success:
-            try:
-                self.tdb.getDB().commit()
-                success = True
-            except sqlite3.OperationalError:
-                print("(commit) Database is locked, waiting for access.", end = "\r")
-                time.sleep(1)
-    
     def importListings(self, listings_file):
         """
-        Updates the market data (AKA the StationItem table) using listings.csv
+        Updates the market data (AKA the StationItem table) using listings_file
         Writes directly to database.
         """
+        listings_path = Path(self.dataPath, listings_file).absolute()
+        from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
+        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
         
-        self.tdenv.NOTE("Processing market data from {}: Start time = {}", listings_file, self.now())
-        if not (self.dataPath / listings_file).exists():
-            self.tdenv.NOTE("File not found, aborting: {}", (self.dataPath / listings_file))
+        total = _count_listing_entries(self.tdenv, listings_path)
+        if not total:
             return
+
+        stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
+        stmt_flush_station   = """DELETE from StationItem WHERE station_id = ?"""
+        stmt_add_listing     = """
+            INSERT OR IGNORE INTO StationItem (
+                station_id, item_id, modified, from_live,
+                demand_price, demand_units, demand_level,
+                supply_price, supply_units, supply_level
+            )
+            VALUES (
+                ?, ?, datetime(?, 'unixepoch'), ?,
+                ?, ?, ?,
+                ?, ?, ?
+            )
+        """
         
-        total = 1
-        
-        from_live = 0 if listings_file == self.listingsPath else 1
-        
-        self.tdenv.DEBUG0(f"Getting total number of entries in {listings_file}...")
-        listings_path = Path(self.dataPath, listings_file)
-        total += file_line_count(listings_path)
-        
-        liveStmt = """UPDATE StationItem
-                    SET from_live = 0
-                    WHERE station_id = ?"""
-        
-        delStmt = "DELETE from StationItem WHERE station_id = ?"
-        
-        listingStmt = """INSERT OR IGNORE INTO StationItem
-                        (station_id, item_id, modified,
-                         demand_price, demand_units, demand_level,
-                         supply_price, supply_units, supply_level, from_live)
-                        VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"""
-        
-        self.tdenv.DEBUG0("Getting list of commodities...")
-        items = [cols[0] for cols in self.execute("SELECT item_id FROM Item ORDER BY item_id")]
-        
-        self.tdenv.DEBUG0("Getting list of stations...")
-        stationList = {
-            stationID
-            for (stationID,) in self.execute("SELECT station_id FROM Station")
-        }
-        
-        stationItems = dict(self.execute('SELECT station_id, UNIXEPOCH(modified) FROM StationItem').fetchall())
+        # Fetch all the items IDS
+        db = self.tdb.getDB()
+        item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
+        station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
+        last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
         
+        cur_station = None
         self.tdenv.DEBUG0("Processing entries...")
         with listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
             prog = pbar.Progress(total, 50)
-            listings = csv.DictReader(fh)
-            
-            cur_station = -1
+
+            cursor: Optional[sqlite3.Cursor] = db.cursor()
             
-            for listing in listings:
-                if prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}"):
-                    # Do a commit and close the DB every 2%.
-                    # This ensures the listings are put in the DB and the WAL is cleared.
-                    self.commit()
-                    self.tdb.close()
+            for listing in csv.DictReader(fh):
+                prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}")
                 
                 station_id = int(listing['station_id'])
-                if station_id not in stationList:
+                if station_id not in station_lookup:
                     continue
                 
+                listing_time = int(listing['collected_at'])
+                
                 if station_id != cur_station:
-                    cur_station = station_id
-                    skipStation = False
+                    # commit anything from the previous station, get a new cursor
+                    db.commit()
+                    cur_station, skip_station, cursor = station_id, False, db.cursor()
                     
                     # Check if listing already exists in DB and needs updated.
-                    if stationItems.get(station_id):
+                    last_modified: int = int(last_station_update_times.get(station_id, 0))
+                    if last_modified:
                         # When the listings.csv data matches the database, update to make from_live == 0.
-                        if int(listing['collected_at']) == stationItems.get(station_id) and not from_live:
-                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live'.")
-                            self.execute(liveStmt, (cur_station,))
+                        if listing_time == last_modified and not from_live:
+                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
+                            cursor.execute(stmt_unliven_station, (cur_station,))
+                            skip_station = True
+                            continue
+
                         # Unless the import file data is newer, nothing else needs to be done for this station,
                         # so the rest of the listings for this station can be skipped.
-                        if int(listing['collected_at']) <= stationItems.get(station_id):
-                            skipStation = True
+                        if listing_time <= last_modified:
+                            skip_station = True
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
-                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station}.")
-                        self.execute(delStmt, (cur_station,))
-                        # We've deleted all the items from this station, so remove it.
-                        del stationItems[station_id]
-
+                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
+                        cursor.execute(stmt_flush_station, (cur_station,))
+                        last_station_update_times[station_id] = listing_time
                 
-                if skipStation:
+                # station skip lasts until we change station id.
+                if skip_station:
                     continue
                 
                 # Since this station is not being skipped, get the data and prepare for insertion into the DB.
                 item_id = int(listing['commodity_id'])
                 # listings.csv includes rare items, which we are ignoring.
-                if item_id not in items:
+                if item_id not in item_lookup:
                     continue
-                modified = datetime.datetime.utcfromtimestamp(int(listing['collected_at'])).strftime('%Y-%m-%d %H:%M:%S')
+
                 demand_price = int(listing['sell_price'])
                 demand_units = int(listing['demand'])
-                demand_level = int(listing['demand_bracket']) if listing['demand_bracket'] != '' else -1
+                demand_level = int(listing.get('demand_bracket') or '-1')
                 supply_price = int(listing['buy_price'])
                 supply_units = int(listing['supply'])
-                supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1
+                supply_level = int(listing.get('supply_bracket') or '-1')
                 
                 self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
-                self.execute(listingStmt, (station_id, item_id, modified,
-                                    demand_price, demand_units, demand_level,
-                                    supply_price, supply_units, supply_level, from_live))
+                cursor.execute(stmt_add_listing, (
+                        station_id, item_id, listing_time, from_live,
+                        demand_price, demand_units, demand_level,
+                        supply_price, supply_units, supply_level,
+                ))
             
-            # Do a final commit to be sure
-            self.commit()
-            self.tdb.close()
-            
-            while prog.value < prog.maxValue:
-                prog.increment(1, postfix = lambda value, total: " " + str(round(value / total * 100)) + "%")
-            prog.clear()
+        prog.clear()
+        
+        # Do a final commit to be sure
+        db.commit()
+        
+        self.tdenv.NOTE("Optimizing database...")
+        db.execute("VACUUM")
+        self.tdb.close()
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
     
@@ -347,14 +343,13 @@ class ImportPlugin(plugins.ImportPluginBase):
         # have been passed, enable 'listings'.
         default = True
         for option in self.options:
-            # if not option in ('force', 'fallback', 'skipvend', 'progbar'):
-            if option not in ('force', 'skipvend'):
+            if option not in ('force', 'skipvend', 'purge'):
                 default = False
         if default:
             self.options["listings"] = True
         
-        # We can probably safely assume that the plugin has never been run if
-        # the prices file doesn't exist, since the plugin always generates it.
+        # We can probably safely assume that the plugin
+        # has never been run if the db file doesn't exist.
         if not (self.tdb.dataPath / Path("TradeDangerous.db")).exists():
             self.options["clean"] = True
         
@@ -497,7 +492,6 @@ class ImportPlugin(plugins.ImportPluginBase):
         
         if self.getOption("purge"):
             self.purgeSystems()
-            # self.commit()
         
         if self.getOption("listings"):
             if self.downloadFile(self.listingsPath) or self.getOption("force"):

tradedangerous/plugins/spansh_plug.py

@@ -1,26 +1,29 @@
+""" Plugin for importing data from spansh """
 from __future__ import annotations
 
+from collections import namedtuple
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from pathlib import Path
+from rich.progress import Progress
 
+from .. import plugins, cache, transfers, csvexport, corrections
+
+import sqlite3
 import sys
 import time
 import typing
-from collections import namedtuple
+
+import ijson
+
 if sys.version_info.major == 3 and sys.version_info.minor >= 10:
     from dataclasses import dataclass
 else:
     dataclass = False  # pylint: disable=invalid-name
 
-from rich.progress import Progress
-import ijson
-import sqlite3
-
-from .. import plugins, cache, transfers, csvexport, corrections
-
 if typing.TYPE_CHECKING:
-    from typing import Any, Iterable, Optional
+    from typing import Any, Optional
+    from collections.abc import Iterable
     from .. tradeenv import TradeEnv
 
 SOURCE_URL = 'https://downloads.spansh.co.uk/galaxy_stations.json'
@@ -237,11 +240,11 @@ class ImportPlugin(plugins.ImportPluginBase):
     def commit(self, *, force: bool = False) -> None:
         """ Perform a commit if required, but try not to do a crazy amount of committing. """
         if not force and not self.need_commit:
-            return self.cursor
+            return
         
         if not force and self.commit_limit > 0:
             self.commit_limit -= 1
-            return self.cursor
+            return
         
         db = self.tdb.getDB()
         db.commit()
@@ -392,7 +395,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             categories.setdefault(commodity.category, []).append(commodity)
         return categories
     
-    def execute(self, query: str, *params, commitable: bool = False) -> Optional[sqlite3.Cursor]:
+    def execute(self, query: str, *params, commitable: bool = False) -> sqlite3.Cursor:
         """ helper method that performs retriable queries and marks the transaction as needing to commit
             if the query is commitable."""
         if commitable:
@@ -404,14 +407,14 @@ class ImportPlugin(plugins.ImportPluginBase):
             except sqlite3.OperationalError as ex:
                 if "no transaction is active" in str(ex):
                     self.print(f"no transaction for {query}")
-                    return
+                    raise
                 if not attempts:
                     raise
                 attempts -= 1
                 self.print(f'Retrying query \'{query}\': {ex!s}')
                 time.sleep(1)
     
-    def executemany(self, query: str, data: Iterable[Any], *, commitable: bool = False) -> Optional[sqlite3.Cursor]:
+    def executemany(self, query: str, data: Iterable[Any], *, commitable: bool = False) -> sqlite3.Cursor:
         """ helper method that performs retriable queries and marks the transaction as needing to commit
             if the query is commitable."""
         if commitable:
@@ -423,7 +426,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             except sqlite3.OperationalError as ex:
                 if "no transaction is active" in str(ex):
                     self.print(f"no transaction for {query}")
-                    return
+                    raise
                 if not attempts:
                     raise
                 attempts -= 1

tradedangerous/submit-distances.py

@@ -16,6 +16,7 @@ import argparse
 import os
 import random
 import re
+import requests
 import sys
 import tradedb
 import tradeenv
@@ -23,30 +24,6 @@ import tradeenv
 from misc.edsc import StarSubmission, StarSubmissionResult, SubmissionError
 from misc.clipboard import SystemNameClip
 
-try:
-    import requests
-except ImportError as e:
-    print("""ERROR: Unable to load the Python 'requests' package.
-
-This script uses a Python module/package called 'requests' to allow
-it to talk to the EDSC web service. This package is not installed
-by default, but it can be installed with Python's package manager (pip).
-
-You can either install/update it yourself, e.g.:
-  
-  pip install --upgrade requests
-
-or if you like, I can try and install it for you now
-""")
-    approval = input(
-        "Do you want me to try and install it with the package manager (y/n)? "
-    )
-    if approval.lower() != 'y':
-        print("You didn't type 'y' so I'm giving up.")
-        raise e
-    import pip
-    pip.main(["install", "--upgrade", "requests"])
-    import requests  # noqa: F401
 
 standardStars = [
     "SOL",

tradedangerous/transfers.py

@@ -1,82 +1,15 @@
+import csv
+import json
+import time
+
 from collections import deque
 from pathlib import Path
 from .tradeexcept import TradeException
-
-import csv
-import json
 from .misc import progress as pbar
-import platform  # noqa: F401
 from . import fs
-import time
-import subprocess
-import sys
 
-try:
-    import requests
-    __requests = requests
-except ImportError:
-    __requests = None
+import requests
 
-def import_requests():
-    global __requests
-    if __requests:
-        return __requests
-    
-    if platform.system() == 'Linux':
-        extra = (
-            "Ubuntu users: You may be able to install 'pip'\n"
-            "with 'apt-get install python3-pip' and requests with\n"
-            "'pip3 install --upgrade requests'.\n"
-        )
-    elif platform.system() == 'Windows':
-        extra = (
-            "\n\nThe requests package can be installed with\n"
-            "'pip3 install --upgrade requests'.\n\n"
-            "If requests is installed, you may have bits\n"
-            "of 32-bit and 64-bit Python installed.\n"
-            "Consider using control panel to uninstall Python,\n"
-            "delete the Python folder (usually C:\\Python34\\),\n"
-            "and then re-install Python.\n"
-        )
-    else:
-        extra = ""
-    
-    print(
-        "ERROR: Unable to load the Python 'requests' package.\n" + extra
-    )
-    approval = input(
-        "I can try and install the package automatically using 'pip'.\n"
-        "Try to install 'requests' now (y/n)? "
-    )
-    # Idiot-proofing: take just the first character in case the user typed
-    # 'YES' (upper or lower case) instead of 'Y'.
-    if approval[0:1].lower() != 'y':
-        raise TradeException("Missing package: 'requests'")
-    
-    try:
-        import pip  # noqa: F401  # pylint: disable=unused-import
-    except ImportError as e:
-        raise TradeException(
-            "Python 3.4.2 includes a package manager called 'pip', "
-            "except it doesn't appear to be installed on your system:\n"
-            "{}{}".format(str(e), extra)
-        ) from None
-    
-    # Let's use "The most reliable approach, and the one that is fully supported."
-    # Especially since the old way produces an error for me on Python 3.6:
-    # "AttributeError: 'module' object has no attribute 'main'"
-    #  pip.main(["install", "--upgrade", "requests"])
-    subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'requests'])
-    
-    try:
-        import requests  # pylint: disable=redefined-outer-name
-        __requests = requests
-    except ImportError as e:
-        raise TradeException(
-            f"The requests module did not install correctly ({e}).{extra}"
-        ) from None
-    
-    return __requests
 
 ######################################################################
 # Helpers
@@ -125,7 +58,6 @@ def download(
         function to call on the first line
     """
     
-    requests = import_requests()  # pylint: disable=redefined-outer-name
     tdenv.NOTE("Requesting {}".format(url))
     req = requests.get(url, headers=headers or None, stream=True, timeout=timeout)
     req.raise_for_status()
@@ -238,7 +170,6 @@ def get_json_data(url, *, timeout: int = 90):
     Displays a progress bar as it downloads.
     """
     
-    requests = import_requests()  # pylint: disable=redefined-outer-name
     req = requests.get(url, stream=True, timeout=timeout)
     
     totalLength = req.headers.get('content-length')
@@ -279,7 +210,6 @@ class CSVStream:
         self.url = url
         self.tdenv = tdenv
         if not url.startswith("file:///"):
-            requests = import_requests()  # pylint: disable=redefined-outer-name
             self.req = requests.get(self.url, stream=True, timeout=timeout)
             self.lines = self.req.iter_lines()
         else:

tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '11.0.2'
+__version__ = '11.0.4'
 

{tradedangerous-11.0.2.dist-info → tradedangerous-11.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 11.0.2
+Version: 11.0.4
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-11.0.2.dist-info → tradedangerous-11.0.4.dist-info}/RECORD

@@ -1,17 +1,17 @@
 tradedangerous/__init__.py,sha256=bwsbE_GyCNsuyGDKnfXAg0RD-ewsWHliySJ5QfCK7h8,1166
-tradedangerous/cache.py,sha256=VH7_Bs-vz1H9KEbQah_4w3zqY3nG7HOegHZGzlKip80,36028
-tradedangerous/cli.py,sha256=4B6FFYyH8M6Z4D8CJHsrkABHsb_O7pmTrJ7M_lB2Za8,4531
+tradedangerous/cache.py,sha256=mROADjtCtSWkvR6crdl7SycBH0AjuaceagUj-x9RZDM,36071
+tradedangerous/cli.py,sha256=dLekZ3MTbn9XcSGtE532qZF3iSnsb5G-ddQyErwTv9o,4559
 tradedangerous/corrections.py,sha256=_WLgo1IBWoskrrPFeshRwCOeJ2BeJb_x4tDQ0JdAo-s,1340
-tradedangerous/csvexport.py,sha256=0XrQKLdt1STinZsBlBN3NHGh0BT81HKVe_oi9lFlLno,8637
+tradedangerous/csvexport.py,sha256=_19bGVnCGJPzvx_8DzLGOs4rqx8Asn7eCVdXKdKu92E,8653
 tradedangerous/edscupdate.py,sha256=G8N-j61MZbCgY07WIVK23CCw6eMPE3WkraXHeARt9QE,17297
 tradedangerous/edsmupdate.py,sha256=jpDUxUkDKxlG_gS3qWy5fg9p3mBPC-akU2pHSlfTPkI,14915
-tradedangerous/formatting.py,sha256=0NbDTQ5fV8x3lhmPTtpnAwQog79dCjPCbDNU5b-Vn6U,6896
+tradedangerous/formatting.py,sha256=-FdlH1yMKPkqhpCtKmbl0chbyZ1VjKZsZ8OLc8v-Knc,6927
 tradedangerous/fs.py,sha256=_5OJHIRy_M-WFHEZO4g2EIgqDxdrz4s8tcdDmgG7NME,2494
 tradedangerous/gui.py,sha256=DFsF5zATr-lyJShL6t5kPKvcLLJYkICurzBz0WBa-oQ,43676
 tradedangerous/jsonprices.py,sha256=GAaNbfEs1LX_F5f69MiEhHIEwoGZtubGC8EQ9MM4B9A,6878
 tradedangerous/mapping.py,sha256=eGBQeYPD04Kq_ygZCDRafKMGz9EnxSgXUzQU-u78_2A,4049
 tradedangerous/prices.py,sha256=JqiDVrtvvPd5pqE3HdwOHOuFgdAbOR-pt0GLD3ZIXM8,7425
-tradedangerous/submit-distances.py,sha256=vBKV2KXz3mudDsuUieIwWjCqo21XdXP9Clb-VufXyUA,12324
+tradedangerous/submit-distances.py,sha256=EMpqxXFauo3oaR9OAnHCHCsrqwABK1nZdmg92m62zrk,11560
 tradedangerous/tools.py,sha256=pp-4WtA12SVaaQHFJFOMTF7EDFRCU2mQeOhC4xoXmEk,1331
 tradedangerous/trade.py,sha256=nGiTFj9ZrqeN9Xad3z8e4MpA2fNqYAcYMrZnwpnuus4,1938
 tradedangerous/tradecalc.py,sha256=A7peEMiaCjlwFvReSq3E7_Ar0shUoFedQi83ZmOc7uY,42075
@@ -19,9 +19,9 @@ tradedangerous/tradedb.py,sha256=3nGB55dYs4igP3U3J4Ye1-M6Kt9A4xPAMmnX7JEDW7w,722
 tradedangerous/tradeenv.py,sha256=SDzRC6ERYZzzb_I6uexmFpFJJrnbzXa-1ogYt_GH26w,10576
 tradedangerous/tradeexcept.py,sha256=aZ-Y31MbkjF7lmAzBAbaMsPPE7FEEfuf4gaX2GvriDk,368
 tradedangerous/tradegui.py,sha256=q2HdIdoyeLUpeF2X0hVIGn7sU6T4zOzq1HN0zGvZdyE,788
-tradedangerous/transfers.py,sha256=NWdrE3gfBlvqnoSJEuWHqYbWFHUHNDd5D0YYMQkpaMY,10513
+tradedangerous/transfers.py,sha256=88gIvXpjd8T6NLxBrBRzmH2IfUmDDtiMcDTrc7qF3OI,7830
 tradedangerous/utils.py,sha256=PUPvAEqUyxYGqqQa0b_yfLAvq8YVUxK6HfdS-CxM-Lo,5186
-tradedangerous/version.py,sha256=WM4xA7PmvxejHn0EFoa9NayVrOkwZG5LJisnSbHdI4o,646
+tradedangerous/version.py,sha256=VIRNV1KJN_g_ByHP19vQGbhe4W2DtWVHoyZrEjvMkaI,646
 tradedangerous/commands/TEMPLATE.py,sha256=MOE69xsZPHPIMBQ-LXicfsOlCZdy-2gPX_nlnwYYil8,2026
 tradedangerous/commands/__init__.py,sha256=3gz2cnXNZNkV1gtZh0dOnCRxBkQHbeIyysRe3bM2WEE,9516
 tradedangerous/commands/buildcache_cmd.py,sha256=_8vKu9e3tQy0HEPrnG8Ts0OoQ_kF6gZPQ9EOfTUd73w,2179
@@ -34,7 +34,7 @@ tradedangerous/commands/local_cmd.py,sha256=tf7YMGX-vaVGNO2lvQF9EvQEN3Wj7DE9-NTS
 tradedangerous/commands/market_cmd.py,sha256=Ig16zDuksywiiF3Exps6UuM-ZhqgbXqkW6Lu2s9xQf0,5411
 tradedangerous/commands/nav_cmd.py,sha256=v245L1MxiUliITUgvWeeB4cL4UdkNO8n0CiP6ztrV54,8460
 tradedangerous/commands/olddata_cmd.py,sha256=6rpPRRs4kLhV9c0sogmctVAjta7v0L0Deu999spXY2s,7888
-tradedangerous/commands/parsing.py,sha256=vyfRByRVLENrtD6IQitdPlddb-f_tgde9arYkimJAEg,6717
+tradedangerous/commands/parsing.py,sha256=RUnCnML59IOXYI6jEljE5lrE_WKcDmFFqKF1JVJg9A0,6725
 tradedangerous/commands/rares_cmd.py,sha256=L_QoW2zIZTU9Jpavt_K2fZyu8T153nUSuVqIiz4uksQ,9207
 tradedangerous/commands/run_cmd.py,sha256=HtvxKfD2ef_fUCDdUzBv9yRwd2gBOqIq4hzUAOLgcyU,47650
 tradedangerous/commands/sell_cmd.py,sha256=RI8T1DWaBuQkYSy2Pjoo0Kxy5n9UgRTqtlSBE0Vg1SI,7943
@@ -54,26 +54,26 @@ tradedangerous/misc/derp-sentinel.py,sha256=22zyl26Q8XHvq8IJZDinJ3oKudUnByDoQ3Wi
 tradedangerous/misc/diff-system-csvs.py,sha256=67K5BEDFYWCSJY17EjkUzNaLX8sOlP9aQ5US6f8WjPw,4931
 tradedangerous/misc/eddb.py,sha256=mHHMIyyuJRHBtBgSou-z7rfNXkajWvvWghRPk3D5jE4,1812
 tradedangerous/misc/eddn.py,sha256=RNoLCxFfEgktMPZB6k2BUxd3aM5Z6LqYi4WhI4jrRdQ,12149
-tradedangerous/misc/edsc.py,sha256=LesWB0M4mMbjsH7la3H5zD-fX-0BL_nIHbLfAjWvy4A,14710
-tradedangerous/misc/edsm.py,sha256=Trp1dNCHgsYppS3r49UZIIDgzantT7WfY3I-eXYj8uA,3253
+tradedangerous/misc/edsc.py,sha256=SN_da9qZ7H8ozibyhoFVB8nAvwBDPF3Z_PMlt70J2Ro,14367
+tradedangerous/misc/edsm.py,sha256=equDwO1SY3QTsoJIb3LjiHes4C8Dejaap_TMpYlCm8o,2910
 tradedangerous/misc/importeddbstats.py,sha256=iLAcrFzdwiMm_MnUuaHcT_xGgZF8pfEEd1qljhhWJTU,1787
 tradedangerous/misc/prices-json-exp.py,sha256=Fpm62ugP35ZBqnRs6ekYfS3GoDFYmvLD3b3SFJfaMOI,4944
 tradedangerous/misc/progress.py,sha256=NKvKP1OSCTpItc1CNxDuEH2A1oGJ6aWSyCdPSAjsG9E,2120
 tradedangerous/plugins/__init__.py,sha256=TL-OIptlqNENKhoFqkFeBJn_vSw8L0pVaDJgjhaTj7A,7860
 tradedangerous/plugins/edapi_plug.py,sha256=5nqBYmjUceAt-KTfiBn7IEl443R1SsGLDmfVXgbcyms,42262
 tradedangerous/plugins/edcd_plug.py,sha256=JuDtuEM_mN9Sz2H09-qYizM-9N3cuNjgvQy7Y-wHwKw,14412
-tradedangerous/plugins/eddblink_plug.py,sha256=pfPROnJKhvLHDNruOx_wHGK876aYS7dE8xcMVuWw1ak,21961
+tradedangerous/plugins/eddblink_plug.py,sha256=EWJrVn5twwPVT-vGicmWuL4NAwf69G_0mw9lnjzn1Xw,21606
 tradedangerous/plugins/edmc_batch_plug.py,sha256=rrP_lFFxWsba8DPEo0WF2EdCiMoRC7tCT8z62MIvtIo,4173
 tradedangerous/plugins/journal_plug.py,sha256=5HMyoxQ7z42qj7NiL8rDxSyTN9gKikoQjyWzJLD-SYQ,23746
 tradedangerous/plugins/netlog_plug.py,sha256=yUl47l9xt3kGj9oSiY_FZaDGdnQj63oa9MBtSeIy1Zo,13469
-tradedangerous/plugins/spansh_plug.py,sha256=viIlFNrVocRKjZ1rg8mNTZsHaZoSAn5_6vEGn67suVQ,26730
+tradedangerous/plugins/spansh_plug.py,sha256=FmOJcYq6-ENYdVNRN96kYdskeqZ4IqzjEsyCYm6hVXI,26762
 tradedangerous/templates/Added.csv,sha256=8o54civQCcS9y7_DBo0GX196XWRbbREQqKDYTKibsgQ,649
 tradedangerous/templates/Category.csv,sha256=8xwUDcBZE25T6x6dZGlRUMTCqeDLt3a9LXU5h6hRHV8,250
 tradedangerous/templates/RareItem.csv,sha256=F1RhRnTD82PiwrVUO-ai2ErGH2PTqNnQaDw5mcgljXs,10483
 tradedangerous/templates/TradeDangerous.sql,sha256=VlQK7QGtEi2brGtWaIZDvKmbJ_vLocD4CJ8h_6kKptU,7808
-tradedangerous-11.0.2.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-tradedangerous-11.0.2.dist-info/METADATA,sha256=_vaFgkOz_3wl328H1FA2rSCzopguYOklEFx_brB23ho,4435
-tradedangerous-11.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tradedangerous-11.0.2.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
-tradedangerous-11.0.2.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
-tradedangerous-11.0.2.dist-info/RECORD,,
+tradedangerous-11.0.4.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+tradedangerous-11.0.4.dist-info/METADATA,sha256=wtvXMMQTXzopIxDvS5GcaFubPfU2pDHlM3T6csHiTlY,4435
+tradedangerous-11.0.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tradedangerous-11.0.4.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
+tradedangerous-11.0.4.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
+tradedangerous-11.0.4.dist-info/RECORD,,