tradedangerous 11.2.0__py3-none-any.whl → 11.3.0__py3-none-any.whl

tradedangerous/cache.py

@@ -30,14 +30,17 @@ import sqlite3
 import sys
 import typing
 
+from functools import partial as partial_fn
+from .fs import file_line_count
 from .tradeexcept import TradeException
+from tradedangerous.misc.progress import Progress, CountingBar
 from . import corrections, utils
 from . import prices
 
 
 # For mypy/pylint type checking
 if typing.TYPE_CHECKING:
-    from typing import Any, Optional, TextIO
+    from typing import Any, Callable, Optional, TextIO  # noqa
     
     from .tradeenv import TradeEnv
 
@@ -777,11 +780,14 @@ def deprecationCheckItem(importPath, lineNo, line):
     )
 
 
-def processImportFile(tdenv, db, importPath, tableName):
+def processImportFile(tdenv, db, importPath, tableName, *, line_callback: Optional[Callable] = None, call_args: Optional[dict] = None):
     tdenv.DEBUG0(
         "Processing import file '{}' for table '{}'",
         str(importPath), tableName
     )
+    call_args = call_args or {}
+    if line_callback:
+        line_callback = partial_fn(line_callback, **call_args)
     
     fkeySelectStr = (
         "("
@@ -870,6 +876,8 @@ def processImportFile(tdenv, db, importPath, tableName):
         uniqueIndex = {}
         
         for linein in csvin:
+            if line_callback:
+                line_callback()
             if not linein:
                 continue
             lineNo = csvin.line_num
@@ -977,25 +985,34 @@ def buildCache(tdb, tdenv):
         tempDB.executescript(sqlScript)
     
     # import standard tables
-    for (importName, importTable) in tdb.importTables:
-        try:
-            processImportFile(tdenv, tempDB, Path(importName), importTable)
-        except FileNotFoundError:
-            tdenv.DEBUG0(
-                "WARNING: processImportFile found no {} file", importName
-            )
-        except StopIteration:
-            tdenv.NOTE(
-                "{} exists but is empty. "
-                "Remove it or add the column definition line.",
-                importName
-            )
-    
-    tempDB.commit()
+    with Progress(max_value=len(tdb.importTables) + 1, prefix="Importing", width=25, style=CountingBar) as prog:
+        for importName, importTable in tdb.importTables:
+            import_path = Path(importName)
+            import_lines = file_line_count(import_path, missing_ok=True)
+            with prog.sub_task(max_value=import_lines, description=importTable) as child:
+                prog.increment(value=1)
+                call_args = {'task': child, 'advance': 1}
+                try:
+                    processImportFile(tdenv, tempDB, import_path, importTable, line_callback=prog.update_task, call_args=call_args)
+                except FileNotFoundError:
+                    tdenv.DEBUG0(
+                        "WARNING: processImportFile found no {} file", importName
+                    )
+                except StopIteration:
+                    tdenv.NOTE(
+                        "{} exists but is empty. "
+                        "Remove it or add the column definition line.",
+                        importName
+                    )
+        prog.increment(1)
+
+        with prog.sub_task(description="Save DB"):
+            tempDB.commit()
     
     # Parse the prices file
     if pricesPath.exists():
-        processPricesFile(tdenv, tempDB, pricesPath)
+        with Progress(max_value=None, width=25, prefix="Processing prices file"):
+            processPricesFile(tdenv, tempDB, pricesPath)
     else:
         tdenv.NOTE(
                 "Missing \"{}\" file - no price data.",

tradedangerous/commands/parsing.py

@@ -60,7 +60,6 @@ class PadSizeArgument(int):
             'dest': 'padSize',
             'metavar': 'PADSIZES',
             'type': 'padsize',
-            'choices': 'SsMmLl?',
         }
 
 
@@ -153,7 +152,6 @@ class PlanetaryArgument(int):
             'dest': 'planetary',
             'metavar': 'PLANETARY',
             'type': 'planetary',
-            'choices': 'YyNn?',
         }
 
 
@@ -181,7 +179,6 @@ class FleetCarrierArgument(int):
             'dest': 'fleet',
             'metavar': 'FLEET',
             'type': 'fleet',
-            'choices': 'YyNn?',
         }
 
 class OdysseyArgument(int):
@@ -208,7 +205,6 @@ class OdysseyArgument(int):
             'dest': 'odyssey',
             'metavar': 'ODYSSEY',
             'type': 'odyssey',
-            'choices': 'YyNn?',
         }
 
 

tradedangerous/edscupdate.py

@@ -1,5 +1,5 @@
-# Deprecated
 #!/usr/bin/env python3.6
+# Deprecated
 
 """
 This tool looks for changes in the EDSC service since the most

tradedangerous/edsmupdate.py

@@ -1,5 +1,5 @@
-# Deprecated
 #!/usr/bin/env python3.6
+# Deprecated
 
 """
 based on edscupdate.py without the submit_distance()

tradedangerous/formatting.py

@@ -8,8 +8,7 @@ import itertools
 import typing
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Optional
-    from collections.abc import Callable
+    from typing import Any, Callable, Optional  # noqa
 
 
 class ColumnFormat:

tradedangerous/fs.py

@@ -1,10 +1,10 @@
 """This module should handle filesystem related operations
 """
 from shutil import copy as shcopy
-from os import makedirs
+from os import makedirs, PathLike
 from pathlib import Path
 
-__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder']
+__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder', 'file_line_count']
 
 def pathify(*args):
     if len(args) > 1 or not isinstance(args[0], Path):
@@ -99,3 +99,39 @@ def ensurefolder(folder):
     except FileExistsError:
         pass
     return folderPath.resolve()
+
+
+def file_line_count(from_file: PathLike, buf_size: int = 128 * 1024, *, missing_ok: bool = False) -> int:
+    """ counts the number of newline characters in a given file. """
+    if not isinstance(from_file, Path):
+        from_file = Path(from_file)
+
+    if missing_ok and not from_file.exists():
+        return 0
+
+    # Pre-allocate a buffer so that we aren't putting pressure on the garbage collector.
+    buf = bytearray(buf_size)
+
+    # Capture it's counting method, so we don't have to keep looking that up on
+    # large files.
+    counter = buf.count
+
+    total = 0
+    with from_file.open("rb") as fh:
+        # Capture the 'readinto' method to avoid lookups.
+        reader = fh.readinto
+
+        # read into the buffer and capture the number of bytes fetched,
+        # which will be 'size' until the last read from the file.
+        read = reader(buf)
+        while read == buf_size:  # nominal case for large files
+            total += counter(b'\n')
+            read = reader(buf)
+
+        # when 0 <= read < buf_size we're on the last page of the
+        # file, so we need to take a slice of the buffer, which creates
+        # a new object, thus we also have to lookup count. it's trivial
+        # but if you have to do it 10,000x it's definitely not a rounding error.
+        return total + buf[:read].count(b'\n')
+
+

tradedangerous/misc/progress.py

@@ -1,71 +1,179 @@
-import sys
+from rich.progress import (
+        Progress as RichProgress,
+        TaskID,
+        ProgressColumn,
+        BarColumn, DownloadColumn, MofNCompleteColumn, SpinnerColumn, 
+        TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn,
+        TransferSpeedColumn
+)
+from contextlib import contextmanager
+
+from typing import Iterable, Optional, Union, Type  # noqa
+
+
+class BarStyle:
+    """ Base class for Progress bar style types. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None, *, add_columns: Optional[Iterable[ProgressColumn]]):
+        self.columns = [SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=width)]
+        if add_columns:
+            self.columns.extend(add_columns)
+
+
+class CountingBar(BarStyle):
+    """ Creates a progress bar that is counting M/N items to completion. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [MofNCompleteColumn(), TimeElapsedColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class DefaultBar(BarStyle):
+    """ Creates a simple default progress bar with a percentage and time elapsed. """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [TaskProgressColumn(), TimeElapsedColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class LongRunningCountBar(BarStyle):
+    """ Creates a progress bar that is counting M/N items to completion with a time-remaining counter """
+    def __init__(self, width: int = 10, prefix: Optional[str] = None):
+        my_columns = [MofNCompleteColumn(), TimeElapsedColumn(), TimeRemainingColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
+
+class TransferBar(BarStyle):
+    """ Creates a progress bar representing a data transfer, which shows the amount of
+        data transferred, speed, and estimated time remaining. """
+    def __init__(self, width: int = 16, prefix: Optional[str] = None):
+        my_columns = [DownloadColumn(), TransferSpeedColumn(), TimeRemainingColumn()]
+        super().__init__(width, prefix, add_columns=my_columns)
+
 
 class Progress:
     """
-    Helper class that describes a simple text-based progress bar.
+    Facade around the rich Progress bar system to help transition away from
+    TD's original basic progress bar implementation.
     """
-    
-    def __init__(self, maxValue, width, start=0, prefix=""):
+    def __init__(self,
+                 max_value: Optional[float] = None,
+                 width: Optional[int] = None,
+                 start: float = 0,
+                 prefix: Optional[str] = None,
+                 *,
+                 style: Optional[Type[BarStyle]] = None,
+                 show: bool = True,
+                 ) -> None:
         """
-        Arguments:
-            maxValue
-                Last value we can reach (100%).
-            width
-                Number of '='s characters for 100%
-            start
-                Initial value
-            prefix
-                Something to print infront of the progress bar
+            :param max_value: Last value we can reach (100%).
+            :param width:     How wide to make the bar itself.
+            :param start:     Override initial value to non-zero.
+            :param prefix:    Text to print between the spinner and the bar.
+            :param style:     Bar-style factory to use for styling.
+            :param show:      If False, disables the bar entirely.
         """
-        self.start = start
-        self.maxValue = maxValue
-        self.width = width
+        self.show = bool(show)
+        if not show:
+            return
+
+        if style is None:
+            style = DefaultBar
+
+        self.max_value = 0 if max_value is None else max(max_value, start)
         self.value = start
-        self.progress = -1
-        self.prefix = prefix
-        self.textLen = 0
-        self.mask = '\r' + self.prefix + "[{{:<{width}}}]".format(width=width)
+        self.prefix = prefix or ""
+        self.width = width or 25
+        # The 'Progress' itself is a view for displaying the progress of tasks. So we construct it
+        # and then create a task for our job.
+        style_instance = style(width=self.width, prefix=self.prefix)
+        self.progress = RichProgress(
+            # What fields to display.
+            *style_instance.columns,
+            # Hide it once it's finished, update it for us, 4x a second
+            transient=True, auto_refresh=True, refresh_per_second=5
+        )
 
+        # Now we add an actual task to track progress on.
+        self.task = self.progress.add_task("Working...", total=max_value, start=True)
+        if self.value:
+            self.progress.update(self.task, advance=self.value)
 
-    def increment(self, value, postfix=""):
-        """
-        Increment the progress bar's internal counter by 'value',
-        and if this changes the progress step, re-draw the bar.
-        
-        Attributes:
-            value
-                The amount to increment the internal counter by
-            postfix [optional]
-                String or callable to print after the bar
+        # And show the task tracker.
+        self.progress.start()
+
+    def __enter__(self):
+        """ Context manager.
         
-        Returns:
-            False if the progress bar did not redraw,
-            True if the progress bar was redrawn,
+            Example use:
+
+                import time
+                import tradedangerous.progress
+
+                # Progress(max_value=100, width=32, style=progress.CountingBar)
+                with progress.Progress(100, 32, style=progress.CountingBar) as prog:
+                    for i in range(100):
+                        prog.increment(1)
+                        time.sleep(3)
         """
-        self.value = min(self.maxValue, self.value + value)
-        progress = int(self.width * (self.value - self.start) / self.maxValue)
-        if progress == self.progress:
-            return False
-        
-        if callable(postfix):
-            postfixText = postfix(self.value, self.maxValue)
-        else:
-            postfixText = postfix
-        text = self.mask.format("="*progress) + postfixText + " "
-        sys.stdout.write(text)
-        sys.stdout.flush()
-        
-        self.progress = progress
-        self.textLen = len(text)
-        
-        return True
+        return self
 
+    def __exit__(self, *args, **kwargs):
+        self.clear()
 
-    def clear(self):
+    def increment(self, value: Optional[float] = None, description: Optional[str] = None, *, progress: Optional[float] = None) -> None:
         """
-        Remove the current progress bar, if any
+        Increase the progress of the bar by a given amount.
+        
+        :param value:  How much to increase the progress by.
+        :param description: If set, replaces the task description.
+        :param progress: Instead of increasing by value, set the absolute progress to this.
         """
-        if self.textLen:
-            fin = "\r{:{width}}\r".format('', width=self.textLen)
-            sys.stdout.write(fin)
-            sys.stdout.flush()
+        if not self.show:
+            return
+        if description:
+            self.prefix = description
+            self.progress.update(self.task, description=description, refresh=True)
+        
+        bump = False
+        if not value and progress is not None and self.value != progress:
+            self.value = progress
+            bump = True
+        elif value:
+            self.value += value              # Update our internal count
+            bump = True
+
+        if self.value >= self.max_value:  # Did we go past the end? Increase the end.
+            self.max_value += value * 2
+            self.progress.update(self.task, description=self.prefix, total=self.max_value)
+            bump = True
+
+        if bump and self.max_value > 0:
+            self.progress.update(self.task, description=self.prefix, completed=self.value)
+
+    def clear(self) -> None:
+        """ Remove the current progress bar, if any. """
+        # These two shouldn't happen separately, but incase someone tinkers, test each
+        # separately and shut them down.
+        if not self.show:
+            return
+
+        if self.task:
+            self.progress.remove_task(self.task)
+            self.task = None
+
+        if self.progress:
+            self.progress.stop()
+            self.progress = None
+            
+    @contextmanager
+    def sub_task(self, description: str, max_value: Optional[int] = None, width: int = 25):
+        if not self.show:
+            yield
+            return
+        task = self.progress.add_task(description, total=max_value, start=True, width=width)
+        try:
+            yield task
+        finally:
+            self.progress.remove_task(task)
+
+    def update_task(self, task: TaskID, advance: Union[float, int], description: Optional[str] = None):
+        if self.show:
+            self.progress.update(task, advance=advance, description=description)