tradedangerous 11.2.1__tar.gz → 11.3.0__tar.gz

{tradedangerous-11.2.1/tradedangerous.egg-info → tradedangerous-11.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 11.2.1
+Version: 11.3.0
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-11.2.1 → tradedangerous-11.3.0}/tradedangerous/cache.py

@@ -30,14 +30,17 @@ import sqlite3
 import sys
 import typing
 
+from functools import partial as partial_fn
+from .fs import file_line_count
 from .tradeexcept import TradeException
+from tradedangerous.misc.progress import Progress, CountingBar
 from . import corrections, utils
 from . import prices
 
 
 # For mypy/pylint type checking
 if typing.TYPE_CHECKING:
-    from typing import Any, Optional, TextIO
+    from typing import Any, Callable, Optional, TextIO  # noqa
     
     from .tradeenv import TradeEnv
 
@@ -777,11 +780,14 @@ def deprecationCheckItem(importPath, lineNo, line):
     )
 
 
-def processImportFile(tdenv, db, importPath, tableName):
+def processImportFile(tdenv, db, importPath, tableName, *, line_callback: Optional[Callable] = None, call_args: Optional[dict] = None):
     tdenv.DEBUG0(
         "Processing import file '{}' for table '{}'",
         str(importPath), tableName
     )
+    call_args = call_args or {}
+    if line_callback:
+        line_callback = partial_fn(line_callback, **call_args)
     
     fkeySelectStr = (
         "("
@@ -870,6 +876,8 @@ def processImportFile(tdenv, db, importPath, tableName):
         uniqueIndex = {}
         
         for linein in csvin:
+            if line_callback:
+                line_callback()
             if not linein:
                 continue
             lineNo = csvin.line_num
@@ -977,25 +985,34 @@ def buildCache(tdb, tdenv):
         tempDB.executescript(sqlScript)
     
     # import standard tables
-    for (importName, importTable) in tdb.importTables:
-        try:
-            processImportFile(tdenv, tempDB, Path(importName), importTable)
-        except FileNotFoundError:
-            tdenv.DEBUG0(
-                "WARNING: processImportFile found no {} file", importName
-            )
-        except StopIteration:
-            tdenv.NOTE(
-                "{} exists but is empty. "
-                "Remove it or add the column definition line.",
-                importName
-            )
-    
-    tempDB.commit()
+    with Progress(max_value=len(tdb.importTables) + 1, prefix="Importing", width=25, style=CountingBar) as prog:
+        for importName, importTable in tdb.importTables:
+            import_path = Path(importName)
+            import_lines = file_line_count(import_path, missing_ok=True)
+            with prog.sub_task(max_value=import_lines, description=importTable) as child:
+                prog.increment(value=1)
+                call_args = {'task': child, 'advance': 1}
+                try:
+                    processImportFile(tdenv, tempDB, import_path, importTable, line_callback=prog.update_task, call_args=call_args)
+                except FileNotFoundError:
+                    tdenv.DEBUG0(
+                        "WARNING: processImportFile found no {} file", importName
+                    )
+                except StopIteration:
+                    tdenv.NOTE(
+                        "{} exists but is empty. "
+                        "Remove it or add the column definition line.",
+                        importName
+                    )
+        prog.increment(1)
+
+        with prog.sub_task(description="Save DB"):
+            tempDB.commit()
     
     # Parse the prices file
     if pricesPath.exists():
-        processPricesFile(tdenv, tempDB, pricesPath)
+        with Progress(max_value=None, width=25, prefix="Processing prices file"):
+            processPricesFile(tdenv, tempDB, pricesPath)
     else:
         tdenv.NOTE(
                 "Missing \"{}\" file - no price data.",

{tradedangerous-11.2.1 → tradedangerous-11.3.0}/tradedangerous/formatting.py

@@ -8,8 +8,7 @@ import itertools
 import typing
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Optional
-    from collections.abc import Callable
+    from typing import Any, Callable, Optional  # noqa
 
 
 class ColumnFormat:

{tradedangerous-11.2.1 → tradedangerous-11.3.0}/tradedangerous/fs.py

@@ -1,10 +1,10 @@
 """This module should handle filesystem related operations
 """
 from shutil import copy as shcopy
-from os import makedirs
+from os import makedirs, PathLike
 from pathlib import Path
 
-__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder']
+__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder', 'file_line_count']
 
 def pathify(*args):
     if len(args) > 1 or not isinstance(args[0], Path):
@@ -99,3 +99,39 @@ def ensurefolder(folder):
     except FileExistsError:
         pass
     return folderPath.resolve()
+
+
+def file_line_count(from_file: PathLike, buf_size: int = 128 * 1024, *, missing_ok: bool = False) -> int:
+    """ counts the number of newline characters in a given file. """
+    if not isinstance(from_file, Path):
+        from_file = Path(from_file)
+
+    if missing_ok and not from_file.exists():
+        return 0
+
+    # Pre-allocate a buffer so that we aren't putting pressure on the garbage collector.
+    buf = bytearray(buf_size)
+
+    # Capture it's counting method, so we don't have to keep looking that up on
+    # large files.
+    counter = buf.count
+
+    total = 0
+    with from_file.open("rb") as fh:
+        # Capture the 'readinto' method to avoid lookups.
+        reader = fh.readinto
+
+        # read into the buffer and capture the number of bytes fetched,
+        # which will be 'size' until the last read from the file.
+        read = reader(buf)
+        while read == buf_size:  # nominal case for large files
+            total += counter(b'\n')
+            read = reader(buf)
+
+        # when 0 <= read < buf_size we're on the last page of the
+        # file, so we need to take a slice of the buffer, which creates
+        # a new object, thus we also have to lookup count. it's trivial
+        # but if you have to do it 10,000x it's definitely not a rounding error.
+        return total + buf[:read].count(b'\n')
+
+

tradedangerous-11.3.0/tradedangerous/misc/progress.py

@@ -0,0 +1,179 @@
+from rich.progress import (
+        Progress as RichProgress,
+        TaskID,
+        ProgressColumn,
+        BarColumn, DownloadColumn, MofNCompleteColumn, SpinnerColumn, 
+        TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn,
+        TransferSpeedColumn
+)
+from contextlib import contextmanager
+
+from typing import Iterable, Optional, Union, Type  # noqa
+
+
+class BarStyle:
+    """ Base class for Progress bar style types. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None, *, add_columns: Optional[Iterable[ProgressColumn]]):
+        self.columns = [SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=width)]
+        if add_columns:
+            self.columns.extend(add_columns)
+
+
+class CountingBar(BarStyle):
+    """ Creates a progress bar that is counting M/N items to completion. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [MofNCompleteColumn(), TimeElapsedColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class DefaultBar(BarStyle):
+    """ Creates a simple default progress bar with a percentage and time elapsed. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [TaskProgressColumn(), TimeElapsedColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class LongRunningCountBar(BarStyle):
+    """ Creates a progress bar that is counting M/N items to completion with a time-remaining counter """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [MofNCompleteColumn(), TimeElapsedColumn(), TimeRemainingColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class TransferBar(BarStyle):
+    """ Creates a progress bar representing a data transfer, which shows the amount of
+        data transferred, speed, and estimated time remaining. """
+    def __init__(self, width: int = 16, prefix: Optional[str] = None):
+        my_columns = [DownloadColumn(), TransferSpeedColumn(), TimeRemainingColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class Progress:
+    """
+    Facade around the rich Progress bar system to help transition away from
+    TD's original basic progress bar implementation.
+    """
+    def __init__(self,
+                 max_value: Optional[float] = None,
+                 width: Optional[int] = None,
+                 start: float = 0,
+                 prefix: Optional[str] = None,
+                 *,
+                 style: Optional[Type[BarStyle]] = None,
+                 show: bool = True,
+                 ) -> None:
+        """
+            :param max_value: Last value we can reach (100%).
+            :param width:     How wide to make the bar itself.
+            :param start:     Override initial value to non-zero.
+            :param prefix:    Text to print between the spinner and the bar.
+            :param style:     Bar-style factory to use for styling.
+            :param show:      If False, disables the bar entirely.
+        """
+        self.show = bool(show)
+        if not show:
+            return
+
+        if style is None:
+            style = DefaultBar
+
+        self.max_value = 0 if max_value is None else max(max_value, start)
+        self.value = start
+        self.prefix = prefix or ""
+        self.width = width or 25
+        # The 'Progress' itself is a view for displaying the progress of tasks. So we construct it
+        # and then create a task for our job.
+        style_instance = style(width=self.width, prefix=self.prefix)
+        self.progress = RichProgress(
+            # What fields to display.
+            *style_instance.columns,
+            # Hide it once it's finished, update it for us, 4x a second
+            transient=True, auto_refresh=True, refresh_per_second=5
+        )
+
+        # Now we add an actual task to track progress on.
+        self.task = self.progress.add_task("Working...", total=max_value, start=True)
+        if self.value:
+            self.progress.update(self.task, advance=self.value)
+
+        # And show the task tracker.
+        self.progress.start()
+
+    def __enter__(self):
+        """ Context manager.
+        
+            Example use:
+
+                import time
+                import tradedangerous.progress
+
+                # Progress(max_value=100, width=32, style=progress.CountingBar)
+                with progress.Progress(100, 32, style=progress.CountingBar) as prog:
+                    for i in range(100):
+                        prog.increment(1)
+                        time.sleep(3)
+        """
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.clear()
+
+    def increment(self, value: Optional[float] = None, description: Optional[str] = None, *, progress: Optional[float] = None) -> None:
+        """
+        Increase the progress of the bar by a given amount.
+        
+        :param value:  How much to increase the progress by.
+        :param description: If set, replaces the task description.
+        :param progress: Instead of increasing by value, set the absolute progress to this.
+        """
+        if not self.show:
+            return
+        if description:
+            self.prefix = description
+            self.progress.update(self.task, description=description, refresh=True)
+        
+        bump = False
+        if not value and progress is not None and self.value != progress:
+            self.value = progress
+            bump = True
+        elif value:
+            self.value += value              # Update our internal count
+            bump = True
+
+        if self.value >= self.max_value:  # Did we go past the end? Increase the end.
+            self.max_value += value * 2
+            self.progress.update(self.task, description=self.prefix, total=self.max_value)
+            bump = True
+
+        if bump and self.max_value > 0:
+            self.progress.update(self.task, description=self.prefix, completed=self.value)
+
+    def clear(self) -> None:
+        """ Remove the current progress bar, if any. """
+        # These two shouldn't happen separately, but incase someone tinkers, test each
+        # separately and shut them down.
+        if not self.show:
+            return
+
+        if self.task:
+            self.progress.remove_task(self.task)
+            self.task = None
+
+        if self.progress:
+            self.progress.stop()
+            self.progress = None
+            
+    @contextmanager
+    def sub_task(self, description: str, max_value: Optional[int] = None, width: int = 25):
+        if not self.show:
+            yield
+            return
+        task = self.progress.add_task(description, total=max_value, start=True, width=width)
+        try:
+            yield task
+        finally:
+            self.progress.remove_task(task)
+
+    def update_task(self, task: TaskID, advance: Union[float, int], description: Optional[str] = None):
+        if self.show:
+            self.progress.update(task, advance=advance, description=description)

{tradedangerous-11.2.1 → tradedangerous-11.3.0}/tradedangerous/plugins/eddblink_plug.py

@@ -4,19 +4,18 @@ https://elite.tromador.com/ to update the Database.
 """
 from __future__ import annotations
 
+from email.utils import parsedate_to_datetime
 from pathlib import Path
+from .. fs import file_line_count
 from .. import plugins, cache, transfers
 from ..misc import progress as pbar
 from ..plugins import PluginException
 
-import certifi
 import csv
 import datetime
-from email.utils import parsedate_to_datetime
 import os
 import requests
 import sqlite3
-import ssl
 import typing
 
 
@@ -26,41 +25,12 @@ if typing.TYPE_CHECKING:
 
 # Constants
 BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
-CONTEXT=ssl.create_default_context(cafile=certifi.where())
 
 
 class DecodingError(PluginException):
     pass
 
 
-def _file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
-    """ counts the number of newline characters in a given file. """
-    # Pre-allocate a buffer so we aren't putting pressure on the garbage collector.
-    buf = bytearray(bufsize)
-    
-    # Capture it's counting method, so we don't have to keep looking that up on
-    # large files.
-    counter = buf.count
-    
-    total = 0
-    with from_file.open("rb") as fh:
-        # Capture the 'readinto' method to avoid lookups.
-        reader = fh.readinto
-
-        # read into the buffer and capture the number of bytes fetched,
-        # which will be 'size' until the last read from the file.
-        read = reader(buf)
-        while read == bufsize:  # nominal case for large files
-            total += counter(b'\n')
-            read = reader(buf)
-
-        # when 0 <= read < bufsize we're on the last page of the
-        # file, so we need to take a slice of the buffer, which creates
-        # a new object and thus we also have to lookup count. it's trivial
-        # but if you have to do it 10,000x it's definitely not a rounding error.
-        return total + buf[:read].count(b'\n')
-
-
 def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
     """ Calculates the number of entries in a listing file by counting the lines. """
     if not listings.exists():
@@ -68,7 +38,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
         return 0
     
     tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
-    count = _file_line_count(listings)
+    count = file_line_count(listings)
     if count <= 1:
         if count == 1:
             tdenv.DEBUG0("Listing count of 1 suggests nothing but a header")
@@ -101,7 +71,6 @@ class ImportPlugin(plugins.ImportPluginBase):
     """
     Plugin that downloads data from eddb.
     """
-    
     pluginOptions = {
         'item':         "Update Items using latest file from server. (Implies '-O system,station')",
         'rare':         "Update RareItems using latest file from server. (Implies '-O system,station')",
@@ -213,12 +182,16 @@ class ImportPlugin(plugins.ImportPluginBase):
         """
         listings_path = Path(self.dataPath, listings_file).absolute()
         from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
-        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
         
+        self.tdenv.NOTE("Checking listings")
         total = _count_listing_entries(self.tdenv, listings_path)
         if not total:
+            self.tdenv.NOTE("No listings")
             return
 
+        self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
+
+        db = self.tdb.getDB()
         stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
         stmt_flush_station   = """DELETE from StationItem WHERE station_id = ?"""
         stmt_add_listing     = """
@@ -235,20 +208,25 @@ class ImportPlugin(plugins.ImportPluginBase):
         """
         
         # Fetch all the items IDS
-        db = self.tdb.getDB()
         item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
         station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
         last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
         
         cur_station = None
+        is_debug = self.tdenv.debug > 0
         self.tdenv.DEBUG0("Processing entries...")
-        with listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
-            prog = pbar.Progress(total, 50)
-
-            cursor: Optional[sqlite3.Cursor] = db.cursor()
+        
+        # Try to find a balance between doing too many commits where we fail
+        # to get any benefits from constructing transactions, and blowing up
+        # the WAL and memory usage by making massive transactions.
+        max_transaction_items, transaction_items = 32 * 1024, 0
+        with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog,\
+              listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
+            cursor = db.cursor()
+            cursor.execute("BEGIN TRANSACTION")
             
             for listing in csv.DictReader(fh):
-                prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}")
+                prog.increment(1)
                 
                 station_id = int(listing['station_id'])
                 if station_id not in station_lookup:
@@ -258,16 +236,21 @@ class ImportPlugin(plugins.ImportPluginBase):
                 
                 if station_id != cur_station:
                     # commit anything from the previous station, get a new cursor
-                    db.commit()
-                    cur_station, skip_station, cursor = station_id, False, db.cursor()
+                    if transaction_items >= max_transaction_items:
+                        cursor.execute("COMMIT")
+                        transaction_items = 0
+                        cursor.execute("BEGIN TRANSACTION")
+                    cur_station, skip_station = station_id, False
                     
                     # Check if listing already exists in DB and needs updated.
                     last_modified: int = int(last_station_update_times.get(station_id, 0))
                     if last_modified:
                         # When the listings.csv data matches the database, update to make from_live == 0.
                         if listing_time == last_modified and not from_live:
-                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
+                            if is_debug:
+                                self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
                             cursor.execute(stmt_unliven_station, (cur_station,))
+                            transaction_items += 1
                             skip_station = True
                             continue
 
@@ -278,8 +261,10 @@ class ImportPlugin(plugins.ImportPluginBase):
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
-                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
+                        if is_debug:
+                            self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
                         cursor.execute(stmt_flush_station, (cur_station,))
+                        transaction_items += 1
                         last_station_update_times[station_id] = listing_time
                 
                 # station skip lasts until we change station id.
@@ -299,20 +284,24 @@ class ImportPlugin(plugins.ImportPluginBase):
                 supply_units = int(listing['supply'])
                 supply_level = int(listing.get('supply_bracket') or '-1')
                 
-                self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
+                if is_debug:
+                    self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
                 cursor.execute(stmt_add_listing, (
                         station_id, item_id, listing_time, from_live,
                         demand_price, demand_units, demand_level,
                         supply_price, supply_units, supply_level,
                 ))
-            
-        prog.clear()
-        
-        # Do a final commit to be sure
-        db.commit()
-        
-        self.tdenv.NOTE("Optimizing database...")
-        db.execute("VACUUM")
+                transaction_items += 1
+
+        # These will take a little while, which has four steps, so we'll make it a counter.
+        with pbar.Progress(1, 40, prefix="Saving"):
+            # Do a final commit to be sure
+            cursor.execute("COMMIT")
+
+        if self.getOption("optimize"):
+            with pbar.Progress(1, 40, prefix="Optimizing"):
+                db.execute("VACUUM")
+
         self.tdb.close()
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())

{tradedangerous-11.2.1 → tradedangerous-11.3.0}/tradedangerous/templates/TradeDangerous.sql

@@ -92,10 +92,9 @@ CREATE TABLE Station
    UNIQUE (station_id),
 
    FOREIGN KEY (system_id) REFERENCES System(system_id)
-    ON UPDATE CASCADE
     ON DELETE CASCADE
- );
-CREATE INDEX idx_station_by_system ON Station (system_id, station_id);
+ ) WITHOUT ROWID;
+CREATE INDEX idx_station_by_system ON Station (system_id);
 CREATE INDEX idx_station_by_name ON Station (name);
 
 
@@ -226,11 +225,45 @@ CREATE TABLE StationItem
     ON UPDATE CASCADE ON DELETE CASCADE,
   FOREIGN KEY (item_id) REFERENCES Item(item_id)
     ON UPDATE CASCADE ON DELETE CASCADE
-);
+) WITHOUT ROWID;
 CREATE INDEX si_mod_stn_itm ON StationItem(modified, station_id, item_id);
 CREATE INDEX si_itm_dmdpr ON StationItem(item_id, demand_price) WHERE demand_price > 0;
 CREATE INDEX si_itm_suppr ON StationItem(item_id, supply_price) WHERE supply_price > 0;
 
+-- Not used yet
+CREATE TABLE IF NOT EXISTS StationDemand
+(
+    station_id      INTEGER NOT NULL,
+    item_id         INTEGER NOT NULL,
+    price           INTEGER NOT NULL,
+    units           INTEGER NOT NULL,
+    level           INTEGER NOT NULL,
+    modified        INTEGER NOT NULL,
+    from_live       INTEGER NOT NULL DEFAULT 0,
+    CONSTRAINT pk_StationDemand PRIMARY KEY (station_id, item_id),
+    CONSTRAINT fk_StationDemand_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
+    CONSTRAINT fk_StationDemand_item_id_Item       FOREIGN KEY (item_id)    REFERENCES Item    (item_id)    ON DELETE CASCADE
+) WITHOUT ROWID;
+DELETE FROM StationDemand;
+CREATE INDEX idx_StationDemand_item ON StationDemand (item_id);
+
+-- Not used yet
+CREATE TABLE IF NOT EXISTS StationSupply
+(
+    station_id      INTEGER NOT NULL,
+    item_id         INTEGER NOT NULL,
+    price           INTEGER NOT NULL,
+    units           INTEGER NOT NULL,
+    level           INTEGER NOT NULL,
+    modified        INTEGER NOT NULL,
+    from_live       INTEGER NOT NULL DEFAULT 0,
+    CONSTRAINT pk_StationSupply PRIMARY KEY (station_id, item_id),
+    CONSTRAINT fk_StationSupply_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
+    CONSTRAINT fk_StationSupply_item_id_Item       FOREIGN KEY (item_id)    REFERENCES Item    (item_id)    ON DELETE CASCADE
+) WITHOUT ROWID;
+DELETE FROM StationSupply;
+CREATE INDEX idx_StationSupply_item ON StationSupply (item_id);
+
 CREATE VIEW StationBuying AS
 SELECT  station_id,
         item_id,