tradedangerous 11.2.1__tar.gz → 11.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tradedangerous might be problematic. Click here for more details.
- {tradedangerous-11.2.1/tradedangerous.egg-info → tradedangerous-11.4.0}/PKG-INFO +1 -1
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/cache.py +35 -18
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/sell_cmd.py +9 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/formatting.py +1 -2
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/fs.py +38 -2
- tradedangerous-11.4.0/tradedangerous/misc/progress.py +179 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/eddblink_plug.py +43 -54
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/templates/TradeDangerous.sql +37 -4
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/tradecalc.py +156 -159
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/transfers.py +26 -60
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/version.py +1 -1
- {tradedangerous-11.2.1 → tradedangerous-11.4.0/tradedangerous.egg-info}/PKG-INFO +1 -1
- tradedangerous-11.2.1/tradedangerous/misc/progress.py +0 -71
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/LICENSE +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/README.md +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/pyproject.toml +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/setup.cfg +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/setup.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_bootstrap_commands.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_bootstrap_plugins.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_cache.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_commands.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_fs.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_peek.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_tools.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_trade.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_trade_run.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tests/test_utils.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/trade.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/__init__.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/cli.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/TEMPLATE.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/__init__.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/buildcache_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/buy_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/commandenv.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/exceptions.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/export_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/import_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/local_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/market_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/nav_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/olddata_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/parsing.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/rares_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/run_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/shipvendor_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/station_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/trade_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/update_cmd.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/commands/update_gui.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/corrections.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/csvexport.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/edscupdate.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/edsmupdate.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/gui.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/jsonprices.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/mapping.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/mfd/__init__.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/mfd/saitek/__init__.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/mfd/saitek/directoutput.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/mfd/saitek/x52pro.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/checkpricebounds.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/clipboard.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/coord64.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/derp-sentinel.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/diff-system-csvs.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/eddb.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/eddn.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/edsc.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/edsm.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/importeddbstats.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/misc/prices-json-exp.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/__init__.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/edapi_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/edcd_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/edmc_batch_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/journal_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/netlog_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/plugins/spansh_plug.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/prices.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/submit-distances.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/templates/Added.csv +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/templates/Category.csv +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/templates/RareItem.csv +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/tools.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/tradedb.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/tradeenv.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/tradeexcept.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous/utils.py +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/SOURCES.txt +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/dependency_links.txt +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/entry_points.txt +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/not-zip-safe +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/requires.txt +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradedangerous.egg-info/top_level.txt +0 -0
- {tradedangerous-11.2.1 → tradedangerous-11.4.0}/tradegui.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tradedangerous
|
|
3
|
-
Version: 11.
|
|
3
|
+
Version: 11.4.0
|
|
4
4
|
Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
|
|
5
5
|
Home-page: https://github.com/eyeonus/Trade-Dangerous
|
|
6
6
|
Author: eyeonus
|
|
@@ -30,14 +30,17 @@ import sqlite3
|
|
|
30
30
|
import sys
|
|
31
31
|
import typing
|
|
32
32
|
|
|
33
|
+
from functools import partial as partial_fn
|
|
34
|
+
from .fs import file_line_count
|
|
33
35
|
from .tradeexcept import TradeException
|
|
36
|
+
from tradedangerous.misc.progress import Progress, CountingBar
|
|
34
37
|
from . import corrections, utils
|
|
35
38
|
from . import prices
|
|
36
39
|
|
|
37
40
|
|
|
38
41
|
# For mypy/pylint type checking
|
|
39
42
|
if typing.TYPE_CHECKING:
|
|
40
|
-
from typing import Any, Optional, TextIO
|
|
43
|
+
from typing import Any, Callable, Optional, TextIO # noqa
|
|
41
44
|
|
|
42
45
|
from .tradeenv import TradeEnv
|
|
43
46
|
|
|
@@ -777,11 +780,14 @@ def deprecationCheckItem(importPath, lineNo, line):
|
|
|
777
780
|
)
|
|
778
781
|
|
|
779
782
|
|
|
780
|
-
def processImportFile(tdenv, db, importPath, tableName):
|
|
783
|
+
def processImportFile(tdenv, db, importPath, tableName, *, line_callback: Optional[Callable] = None, call_args: Optional[dict] = None):
|
|
781
784
|
tdenv.DEBUG0(
|
|
782
785
|
"Processing import file '{}' for table '{}'",
|
|
783
786
|
str(importPath), tableName
|
|
784
787
|
)
|
|
788
|
+
call_args = call_args or {}
|
|
789
|
+
if line_callback:
|
|
790
|
+
line_callback = partial_fn(line_callback, **call_args)
|
|
785
791
|
|
|
786
792
|
fkeySelectStr = (
|
|
787
793
|
"("
|
|
@@ -870,6 +876,8 @@ def processImportFile(tdenv, db, importPath, tableName):
|
|
|
870
876
|
uniqueIndex = {}
|
|
871
877
|
|
|
872
878
|
for linein in csvin:
|
|
879
|
+
if line_callback:
|
|
880
|
+
line_callback()
|
|
873
881
|
if not linein:
|
|
874
882
|
continue
|
|
875
883
|
lineNo = csvin.line_num
|
|
@@ -977,25 +985,34 @@ def buildCache(tdb, tdenv):
|
|
|
977
985
|
tempDB.executescript(sqlScript)
|
|
978
986
|
|
|
979
987
|
# import standard tables
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
988
|
+
with Progress(max_value=len(tdb.importTables) + 1, prefix="Importing", width=25, style=CountingBar) as prog:
|
|
989
|
+
for importName, importTable in tdb.importTables:
|
|
990
|
+
import_path = Path(importName)
|
|
991
|
+
import_lines = file_line_count(import_path, missing_ok=True)
|
|
992
|
+
with prog.sub_task(max_value=import_lines, description=importTable) as child:
|
|
993
|
+
prog.increment(value=1)
|
|
994
|
+
call_args = {'task': child, 'advance': 1}
|
|
995
|
+
try:
|
|
996
|
+
processImportFile(tdenv, tempDB, import_path, importTable, line_callback=prog.update_task, call_args=call_args)
|
|
997
|
+
except FileNotFoundError:
|
|
998
|
+
tdenv.DEBUG0(
|
|
999
|
+
"WARNING: processImportFile found no {} file", importName
|
|
1000
|
+
)
|
|
1001
|
+
except StopIteration:
|
|
1002
|
+
tdenv.NOTE(
|
|
1003
|
+
"{} exists but is empty. "
|
|
1004
|
+
"Remove it or add the column definition line.",
|
|
1005
|
+
importName
|
|
1006
|
+
)
|
|
1007
|
+
prog.increment(1)
|
|
1008
|
+
|
|
1009
|
+
with prog.sub_task(description="Save DB"):
|
|
1010
|
+
tempDB.commit()
|
|
995
1011
|
|
|
996
1012
|
# Parse the prices file
|
|
997
1013
|
if pricesPath.exists():
|
|
998
|
-
|
|
1014
|
+
with Progress(max_value=None, width=25, prefix="Processing prices file"):
|
|
1015
|
+
processPricesFile(tdenv, tempDB, pricesPath)
|
|
999
1016
|
else:
|
|
1000
1017
|
tdenv.NOTE(
|
|
1001
1018
|
"Missing \"{}\" file - no price data.",
|
|
@@ -37,6 +37,12 @@ switches = [
|
|
|
37
37
|
metavar='N.NN',
|
|
38
38
|
type=float,
|
|
39
39
|
),
|
|
40
|
+
ParseArgument('--age', '--max-days-old', '-MD',
|
|
41
|
+
help = 'Maximum age (in days) of trade data to use.',
|
|
42
|
+
metavar = 'DAYS',
|
|
43
|
+
type = float,
|
|
44
|
+
dest = 'maxAge',
|
|
45
|
+
),
|
|
40
46
|
AvoidPlacesArgument(),
|
|
41
47
|
PadSizeArgument(),
|
|
42
48
|
MutuallyExclusiveGroup(
|
|
@@ -165,6 +171,9 @@ def run(results, cmdenv, tdb: TradeDB):
|
|
|
165
171
|
continue
|
|
166
172
|
if station.system in avoidSystems:
|
|
167
173
|
continue
|
|
174
|
+
maxAge, stnAge = cmdenv.maxAge, station.dataAge or float("inf")
|
|
175
|
+
if maxAge and stnAge > maxAge:
|
|
176
|
+
continue
|
|
168
177
|
|
|
169
178
|
row = ResultRow()
|
|
170
179
|
row.station = station
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""This module should handle filesystem related operations
|
|
2
2
|
"""
|
|
3
3
|
from shutil import copy as shcopy
|
|
4
|
-
from os import makedirs
|
|
4
|
+
from os import makedirs, PathLike
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
|
-
__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder']
|
|
7
|
+
__all__ = ['copy', 'copyallfiles', 'touch', 'ensurefolder', 'file_line_count']
|
|
8
8
|
|
|
9
9
|
def pathify(*args):
|
|
10
10
|
if len(args) > 1 or not isinstance(args[0], Path):
|
|
@@ -99,3 +99,39 @@ def ensurefolder(folder):
|
|
|
99
99
|
except FileExistsError:
|
|
100
100
|
pass
|
|
101
101
|
return folderPath.resolve()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def file_line_count(from_file: PathLike, buf_size: int = 128 * 1024, *, missing_ok: bool = False) -> int:
|
|
105
|
+
""" counts the number of newline characters in a given file. """
|
|
106
|
+
if not isinstance(from_file, Path):
|
|
107
|
+
from_file = Path(from_file)
|
|
108
|
+
|
|
109
|
+
if missing_ok and not from_file.exists():
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
# Pre-allocate a buffer so that we aren't putting pressure on the garbage collector.
|
|
113
|
+
buf = bytearray(buf_size)
|
|
114
|
+
|
|
115
|
+
# Capture it's counting method, so we don't have to keep looking that up on
|
|
116
|
+
# large files.
|
|
117
|
+
counter = buf.count
|
|
118
|
+
|
|
119
|
+
total = 0
|
|
120
|
+
with from_file.open("rb") as fh:
|
|
121
|
+
# Capture the 'readinto' method to avoid lookups.
|
|
122
|
+
reader = fh.readinto
|
|
123
|
+
|
|
124
|
+
# read into the buffer and capture the number of bytes fetched,
|
|
125
|
+
# which will be 'size' until the last read from the file.
|
|
126
|
+
read = reader(buf)
|
|
127
|
+
while read == buf_size: # nominal case for large files
|
|
128
|
+
total += counter(b'\n')
|
|
129
|
+
read = reader(buf)
|
|
130
|
+
|
|
131
|
+
# when 0 <= read < buf_size we're on the last page of the
|
|
132
|
+
# file, so we need to take a slice of the buffer, which creates
|
|
133
|
+
# a new object, thus we also have to lookup count. it's trivial
|
|
134
|
+
# but if you have to do it 10,000x it's definitely not a rounding error.
|
|
135
|
+
return total + buf[:read].count(b'\n')
|
|
136
|
+
|
|
137
|
+
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
from rich.progress import (
|
|
2
|
+
Progress as RichProgress,
|
|
3
|
+
TaskID,
|
|
4
|
+
ProgressColumn,
|
|
5
|
+
BarColumn, DownloadColumn, MofNCompleteColumn, SpinnerColumn,
|
|
6
|
+
TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn,
|
|
7
|
+
TransferSpeedColumn
|
|
8
|
+
)
|
|
9
|
+
from contextlib import contextmanager
|
|
10
|
+
|
|
11
|
+
from typing import Iterable, Optional, Union, Type # noqa
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BarStyle:
|
|
15
|
+
""" Base class for Progress bar style types. """
|
|
16
|
+
def __init__(self, width: int = 10, prefix: Optional[str] = None, *, add_columns: Optional[Iterable[ProgressColumn]]):
|
|
17
|
+
self.columns = [SpinnerColumn(), TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=width)]
|
|
18
|
+
if add_columns:
|
|
19
|
+
self.columns.extend(add_columns)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class CountingBar(BarStyle):
|
|
23
|
+
""" Creates a progress bar that is counting M/N items to completion. """
|
|
24
|
+
def __init__(self, width: int = 10, prefix: Optional[str] = None):
|
|
25
|
+
my_columns = [MofNCompleteColumn(), TimeElapsedColumn()]
|
|
26
|
+
super().__init__(width, prefix, add_columns=my_columns)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DefaultBar(BarStyle):
|
|
30
|
+
""" Creates a simple default progress bar with a percentage and time elapsed. """
|
|
31
|
+
def __init__(self, width: int = 10, prefix: Optional[str] = None):
|
|
32
|
+
my_columns = [TaskProgressColumn(), TimeElapsedColumn()]
|
|
33
|
+
super().__init__(width, prefix, add_columns=my_columns)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LongRunningCountBar(BarStyle):
|
|
37
|
+
""" Creates a progress bar that is counting M/N items to completion with a time-remaining counter """
|
|
38
|
+
def __init__(self, width: int = 10, prefix: Optional[str] = None):
|
|
39
|
+
my_columns = [MofNCompleteColumn(), TimeElapsedColumn(), TimeRemainingColumn()]
|
|
40
|
+
super().__init__(width, prefix, add_columns=my_columns)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TransferBar(BarStyle):
|
|
44
|
+
""" Creates a progress bar representing a data transfer, which shows the amount of
|
|
45
|
+
data transferred, speed, and estimated time remaining. """
|
|
46
|
+
def __init__(self, width: int = 16, prefix: Optional[str] = None):
|
|
47
|
+
my_columns = [DownloadColumn(), TransferSpeedColumn(), TimeRemainingColumn()]
|
|
48
|
+
super().__init__(width, prefix, add_columns=my_columns)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Progress:
|
|
52
|
+
"""
|
|
53
|
+
Facade around the rich Progress bar system to help transition away from
|
|
54
|
+
TD's original basic progress bar implementation.
|
|
55
|
+
"""
|
|
56
|
+
def __init__(self,
|
|
57
|
+
max_value: Optional[float] = None,
|
|
58
|
+
width: Optional[int] = None,
|
|
59
|
+
start: float = 0,
|
|
60
|
+
prefix: Optional[str] = None,
|
|
61
|
+
*,
|
|
62
|
+
style: Optional[Type[BarStyle]] = None,
|
|
63
|
+
show: bool = True,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""
|
|
66
|
+
:param max_value: Last value we can reach (100%).
|
|
67
|
+
:param width: How wide to make the bar itself.
|
|
68
|
+
:param start: Override initial value to non-zero.
|
|
69
|
+
:param prefix: Text to print between the spinner and the bar.
|
|
70
|
+
:param style: Bar-style factory to use for styling.
|
|
71
|
+
:param show: If False, disables the bar entirely.
|
|
72
|
+
"""
|
|
73
|
+
self.show = bool(show)
|
|
74
|
+
if not show:
|
|
75
|
+
return
|
|
76
|
+
|
|
77
|
+
if style is None:
|
|
78
|
+
style = DefaultBar
|
|
79
|
+
|
|
80
|
+
self.max_value = 0 if max_value is None else max(max_value, start)
|
|
81
|
+
self.value = start
|
|
82
|
+
self.prefix = prefix or ""
|
|
83
|
+
self.width = width or 25
|
|
84
|
+
# The 'Progress' itself is a view for displaying the progress of tasks. So we construct it
|
|
85
|
+
# and then create a task for our job.
|
|
86
|
+
style_instance = style(width=self.width, prefix=self.prefix)
|
|
87
|
+
self.progress = RichProgress(
|
|
88
|
+
# What fields to display.
|
|
89
|
+
*style_instance.columns,
|
|
90
|
+
# Hide it once it's finished, update it for us, 4x a second
|
|
91
|
+
transient=True, auto_refresh=True, refresh_per_second=5
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Now we add an actual task to track progress on.
|
|
95
|
+
self.task = self.progress.add_task("Working...", total=max_value, start=True)
|
|
96
|
+
if self.value:
|
|
97
|
+
self.progress.update(self.task, advance=self.value)
|
|
98
|
+
|
|
99
|
+
# And show the task tracker.
|
|
100
|
+
self.progress.start()
|
|
101
|
+
|
|
102
|
+
def __enter__(self):
|
|
103
|
+
""" Context manager.
|
|
104
|
+
|
|
105
|
+
Example use:
|
|
106
|
+
|
|
107
|
+
import time
|
|
108
|
+
import tradedangerous.progress
|
|
109
|
+
|
|
110
|
+
# Progress(max_value=100, width=32, style=progress.CountingBar)
|
|
111
|
+
with progress.Progress(100, 32, style=progress.CountingBar) as prog:
|
|
112
|
+
for i in range(100):
|
|
113
|
+
prog.increment(1)
|
|
114
|
+
time.sleep(3)
|
|
115
|
+
"""
|
|
116
|
+
return self
|
|
117
|
+
|
|
118
|
+
def __exit__(self, *args, **kwargs):
|
|
119
|
+
self.clear()
|
|
120
|
+
|
|
121
|
+
def increment(self, value: Optional[float] = None, description: Optional[str] = None, *, progress: Optional[float] = None) -> None:
|
|
122
|
+
"""
|
|
123
|
+
Increase the progress of the bar by a given amount.
|
|
124
|
+
|
|
125
|
+
:param value: How much to increase the progress by.
|
|
126
|
+
:param description: If set, replaces the task description.
|
|
127
|
+
:param progress: Instead of increasing by value, set the absolute progress to this.
|
|
128
|
+
"""
|
|
129
|
+
if not self.show:
|
|
130
|
+
return
|
|
131
|
+
if description:
|
|
132
|
+
self.prefix = description
|
|
133
|
+
self.progress.update(self.task, description=description, refresh=True)
|
|
134
|
+
|
|
135
|
+
bump = False
|
|
136
|
+
if not value and progress is not None and self.value != progress:
|
|
137
|
+
self.value = progress
|
|
138
|
+
bump = True
|
|
139
|
+
elif value:
|
|
140
|
+
self.value += value # Update our internal count
|
|
141
|
+
bump = True
|
|
142
|
+
|
|
143
|
+
if self.value >= self.max_value: # Did we go past the end? Increase the end.
|
|
144
|
+
self.max_value += value * 2
|
|
145
|
+
self.progress.update(self.task, description=self.prefix, total=self.max_value)
|
|
146
|
+
bump = True
|
|
147
|
+
|
|
148
|
+
if bump and self.max_value > 0:
|
|
149
|
+
self.progress.update(self.task, description=self.prefix, completed=self.value)
|
|
150
|
+
|
|
151
|
+
def clear(self) -> None:
|
|
152
|
+
""" Remove the current progress bar, if any. """
|
|
153
|
+
# These two shouldn't happen separately, but incase someone tinkers, test each
|
|
154
|
+
# separately and shut them down.
|
|
155
|
+
if not self.show:
|
|
156
|
+
return
|
|
157
|
+
|
|
158
|
+
if self.task:
|
|
159
|
+
self.progress.remove_task(self.task)
|
|
160
|
+
self.task = None
|
|
161
|
+
|
|
162
|
+
if self.progress:
|
|
163
|
+
self.progress.stop()
|
|
164
|
+
self.progress = None
|
|
165
|
+
|
|
166
|
+
@contextmanager
|
|
167
|
+
def sub_task(self, description: str, max_value: Optional[int] = None, width: int = 25):
|
|
168
|
+
if not self.show:
|
|
169
|
+
yield
|
|
170
|
+
return
|
|
171
|
+
task = self.progress.add_task(description, total=max_value, start=True, width=width)
|
|
172
|
+
try:
|
|
173
|
+
yield task
|
|
174
|
+
finally:
|
|
175
|
+
self.progress.remove_task(task)
|
|
176
|
+
|
|
177
|
+
def update_task(self, task: TaskID, advance: Union[float, int], description: Optional[str] = None):
|
|
178
|
+
if self.show:
|
|
179
|
+
self.progress.update(task, advance=advance, description=description)
|
|
@@ -4,19 +4,18 @@ https://elite.tromador.com/ to update the Database.
|
|
|
4
4
|
"""
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
from email.utils import parsedate_to_datetime
|
|
7
8
|
from pathlib import Path
|
|
9
|
+
from .. fs import file_line_count
|
|
8
10
|
from .. import plugins, cache, transfers
|
|
9
11
|
from ..misc import progress as pbar
|
|
10
12
|
from ..plugins import PluginException
|
|
11
13
|
|
|
12
|
-
import certifi
|
|
13
14
|
import csv
|
|
14
15
|
import datetime
|
|
15
|
-
from email.utils import parsedate_to_datetime
|
|
16
16
|
import os
|
|
17
17
|
import requests
|
|
18
18
|
import sqlite3
|
|
19
|
-
import ssl
|
|
20
19
|
import typing
|
|
21
20
|
|
|
22
21
|
|
|
@@ -26,41 +25,12 @@ if typing.TYPE_CHECKING:
|
|
|
26
25
|
|
|
27
26
|
# Constants
|
|
28
27
|
BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
|
|
29
|
-
CONTEXT=ssl.create_default_context(cafile=certifi.where())
|
|
30
28
|
|
|
31
29
|
|
|
32
30
|
class DecodingError(PluginException):
|
|
33
31
|
pass
|
|
34
32
|
|
|
35
33
|
|
|
36
|
-
def _file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
|
|
37
|
-
""" counts the number of newline characters in a given file. """
|
|
38
|
-
# Pre-allocate a buffer so we aren't putting pressure on the garbage collector.
|
|
39
|
-
buf = bytearray(bufsize)
|
|
40
|
-
|
|
41
|
-
# Capture it's counting method, so we don't have to keep looking that up on
|
|
42
|
-
# large files.
|
|
43
|
-
counter = buf.count
|
|
44
|
-
|
|
45
|
-
total = 0
|
|
46
|
-
with from_file.open("rb") as fh:
|
|
47
|
-
# Capture the 'readinto' method to avoid lookups.
|
|
48
|
-
reader = fh.readinto
|
|
49
|
-
|
|
50
|
-
# read into the buffer and capture the number of bytes fetched,
|
|
51
|
-
# which will be 'size' until the last read from the file.
|
|
52
|
-
read = reader(buf)
|
|
53
|
-
while read == bufsize: # nominal case for large files
|
|
54
|
-
total += counter(b'\n')
|
|
55
|
-
read = reader(buf)
|
|
56
|
-
|
|
57
|
-
# when 0 <= read < bufsize we're on the last page of the
|
|
58
|
-
# file, so we need to take a slice of the buffer, which creates
|
|
59
|
-
# a new object and thus we also have to lookup count. it's trivial
|
|
60
|
-
# but if you have to do it 10,000x it's definitely not a rounding error.
|
|
61
|
-
return total + buf[:read].count(b'\n')
|
|
62
|
-
|
|
63
|
-
|
|
64
34
|
def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
|
|
65
35
|
""" Calculates the number of entries in a listing file by counting the lines. """
|
|
66
36
|
if not listings.exists():
|
|
@@ -68,7 +38,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
|
|
|
68
38
|
return 0
|
|
69
39
|
|
|
70
40
|
tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
|
|
71
|
-
count =
|
|
41
|
+
count = file_line_count(listings)
|
|
72
42
|
if count <= 1:
|
|
73
43
|
if count == 1:
|
|
74
44
|
tdenv.DEBUG0("Listing count of 1 suggests nothing but a header")
|
|
@@ -101,7 +71,6 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
101
71
|
"""
|
|
102
72
|
Plugin that downloads data from eddb.
|
|
103
73
|
"""
|
|
104
|
-
|
|
105
74
|
pluginOptions = {
|
|
106
75
|
'item': "Update Items using latest file from server. (Implies '-O system,station')",
|
|
107
76
|
'rare': "Update RareItems using latest file from server. (Implies '-O system,station')",
|
|
@@ -213,12 +182,16 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
213
182
|
"""
|
|
214
183
|
listings_path = Path(self.dataPath, listings_file).absolute()
|
|
215
184
|
from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
|
|
216
|
-
self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
|
|
217
185
|
|
|
186
|
+
self.tdenv.NOTE("Checking listings")
|
|
218
187
|
total = _count_listing_entries(self.tdenv, listings_path)
|
|
219
188
|
if not total:
|
|
189
|
+
self.tdenv.NOTE("No listings")
|
|
220
190
|
return
|
|
221
191
|
|
|
192
|
+
self.tdenv.NOTE("Processing market data from {}: Start time = {}. Live = {}", listings_file, self.now(), from_live)
|
|
193
|
+
|
|
194
|
+
db = self.tdb.getDB()
|
|
222
195
|
stmt_unliven_station = """UPDATE StationItem SET from_live = 0 WHERE station_id = ?"""
|
|
223
196
|
stmt_flush_station = """DELETE from StationItem WHERE station_id = ?"""
|
|
224
197
|
stmt_add_listing = """
|
|
@@ -235,20 +208,25 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
235
208
|
"""
|
|
236
209
|
|
|
237
210
|
# Fetch all the items IDS
|
|
238
|
-
db = self.tdb.getDB()
|
|
239
211
|
item_lookup = _make_item_id_lookup(self.tdenv, db.cursor())
|
|
240
212
|
station_lookup = _make_station_id_lookup(self.tdenv, db.cursor())
|
|
241
213
|
last_station_update_times = _collect_station_modified_times(self.tdenv, db.cursor())
|
|
242
214
|
|
|
243
215
|
cur_station = None
|
|
216
|
+
is_debug = self.tdenv.debug > 0
|
|
244
217
|
self.tdenv.DEBUG0("Processing entries...")
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
218
|
+
|
|
219
|
+
# Try to find a balance between doing too many commits where we fail
|
|
220
|
+
# to get any benefits from constructing transactions, and blowing up
|
|
221
|
+
# the WAL and memory usage by making massive transactions.
|
|
222
|
+
max_transaction_items, transaction_items = 32 * 1024, 0
|
|
223
|
+
with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog,\
|
|
224
|
+
listings_path.open("r", encoding="utf-8", errors="ignore") as fh:
|
|
225
|
+
cursor = db.cursor()
|
|
226
|
+
cursor.execute("BEGIN TRANSACTION")
|
|
249
227
|
|
|
250
228
|
for listing in csv.DictReader(fh):
|
|
251
|
-
prog.increment(1
|
|
229
|
+
prog.increment(1)
|
|
252
230
|
|
|
253
231
|
station_id = int(listing['station_id'])
|
|
254
232
|
if station_id not in station_lookup:
|
|
@@ -258,16 +236,21 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
258
236
|
|
|
259
237
|
if station_id != cur_station:
|
|
260
238
|
# commit anything from the previous station, get a new cursor
|
|
261
|
-
|
|
262
|
-
|
|
239
|
+
if transaction_items >= max_transaction_items:
|
|
240
|
+
cursor.execute("COMMIT")
|
|
241
|
+
transaction_items = 0
|
|
242
|
+
cursor.execute("BEGIN TRANSACTION")
|
|
243
|
+
cur_station, skip_station = station_id, False
|
|
263
244
|
|
|
264
245
|
# Check if listing already exists in DB and needs updated.
|
|
265
246
|
last_modified: int = int(last_station_update_times.get(station_id, 0))
|
|
266
247
|
if last_modified:
|
|
267
248
|
# When the listings.csv data matches the database, update to make from_live == 0.
|
|
268
249
|
if listing_time == last_modified and not from_live:
|
|
269
|
-
|
|
250
|
+
if is_debug:
|
|
251
|
+
self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live' (old={last_modified}, listing={listing_time}).")
|
|
270
252
|
cursor.execute(stmt_unliven_station, (cur_station,))
|
|
253
|
+
transaction_items += 1
|
|
271
254
|
skip_station = True
|
|
272
255
|
continue
|
|
273
256
|
|
|
@@ -278,8 +261,10 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
278
261
|
continue
|
|
279
262
|
|
|
280
263
|
# The data from the import file is newer, so we need to delete the old data for this station.
|
|
281
|
-
|
|
264
|
+
if is_debug:
|
|
265
|
+
self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station} (old={last_modified}, listing={listing_time}).")
|
|
282
266
|
cursor.execute(stmt_flush_station, (cur_station,))
|
|
267
|
+
transaction_items += 1
|
|
283
268
|
last_station_update_times[station_id] = listing_time
|
|
284
269
|
|
|
285
270
|
# station skip lasts until we change station id.
|
|
@@ -299,20 +284,24 @@ class ImportPlugin(plugins.ImportPluginBase):
|
|
|
299
284
|
supply_units = int(listing['supply'])
|
|
300
285
|
supply_level = int(listing.get('supply_bracket') or '-1')
|
|
301
286
|
|
|
302
|
-
|
|
287
|
+
if is_debug:
|
|
288
|
+
self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
|
|
303
289
|
cursor.execute(stmt_add_listing, (
|
|
304
290
|
station_id, item_id, listing_time, from_live,
|
|
305
291
|
demand_price, demand_units, demand_level,
|
|
306
292
|
supply_price, supply_units, supply_level,
|
|
307
293
|
))
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
294
|
+
transaction_items += 1
|
|
295
|
+
|
|
296
|
+
# These will take a little while, which has four steps, so we'll make it a counter.
|
|
297
|
+
with pbar.Progress(1, 40, prefix="Saving"):
|
|
298
|
+
# Do a final commit to be sure
|
|
299
|
+
cursor.execute("COMMIT")
|
|
300
|
+
|
|
301
|
+
if self.getOption("optimize"):
|
|
302
|
+
with pbar.Progress(1, 40, prefix="Optimizing"):
|
|
303
|
+
db.execute("VACUUM")
|
|
304
|
+
|
|
316
305
|
self.tdb.close()
|
|
317
306
|
|
|
318
307
|
self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
|
|
@@ -92,10 +92,9 @@ CREATE TABLE Station
|
|
|
92
92
|
UNIQUE (station_id),
|
|
93
93
|
|
|
94
94
|
FOREIGN KEY (system_id) REFERENCES System(system_id)
|
|
95
|
-
ON UPDATE CASCADE
|
|
96
95
|
ON DELETE CASCADE
|
|
97
|
-
);
|
|
98
|
-
CREATE INDEX idx_station_by_system ON Station (system_id
|
|
96
|
+
) WITHOUT ROWID;
|
|
97
|
+
CREATE INDEX idx_station_by_system ON Station (system_id);
|
|
99
98
|
CREATE INDEX idx_station_by_name ON Station (name);
|
|
100
99
|
|
|
101
100
|
|
|
@@ -226,11 +225,45 @@ CREATE TABLE StationItem
|
|
|
226
225
|
ON UPDATE CASCADE ON DELETE CASCADE,
|
|
227
226
|
FOREIGN KEY (item_id) REFERENCES Item(item_id)
|
|
228
227
|
ON UPDATE CASCADE ON DELETE CASCADE
|
|
229
|
-
);
|
|
228
|
+
) WITHOUT ROWID;
|
|
230
229
|
CREATE INDEX si_mod_stn_itm ON StationItem(modified, station_id, item_id);
|
|
231
230
|
CREATE INDEX si_itm_dmdpr ON StationItem(item_id, demand_price) WHERE demand_price > 0;
|
|
232
231
|
CREATE INDEX si_itm_suppr ON StationItem(item_id, supply_price) WHERE supply_price > 0;
|
|
233
232
|
|
|
233
|
+
-- Not used yet
|
|
234
|
+
CREATE TABLE IF NOT EXISTS StationDemand
|
|
235
|
+
(
|
|
236
|
+
station_id INTEGER NOT NULL,
|
|
237
|
+
item_id INTEGER NOT NULL,
|
|
238
|
+
price INTEGER NOT NULL,
|
|
239
|
+
units INTEGER NOT NULL,
|
|
240
|
+
level INTEGER NOT NULL,
|
|
241
|
+
modified INTEGER NOT NULL,
|
|
242
|
+
from_live INTEGER NOT NULL DEFAULT 0,
|
|
243
|
+
CONSTRAINT pk_StationDemand PRIMARY KEY (station_id, item_id),
|
|
244
|
+
CONSTRAINT fk_StationDemand_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
|
|
245
|
+
CONSTRAINT fk_StationDemand_item_id_Item FOREIGN KEY (item_id) REFERENCES Item (item_id) ON DELETE CASCADE
|
|
246
|
+
) WITHOUT ROWID;
|
|
247
|
+
DELETE FROM StationDemand;
|
|
248
|
+
CREATE INDEX idx_StationDemand_item ON StationDemand (item_id);
|
|
249
|
+
|
|
250
|
+
-- Not used yet
|
|
251
|
+
CREATE TABLE IF NOT EXISTS StationSupply
|
|
252
|
+
(
|
|
253
|
+
station_id INTEGER NOT NULL,
|
|
254
|
+
item_id INTEGER NOT NULL,
|
|
255
|
+
price INTEGER NOT NULL,
|
|
256
|
+
units INTEGER NOT NULL,
|
|
257
|
+
level INTEGER NOT NULL,
|
|
258
|
+
modified INTEGER NOT NULL,
|
|
259
|
+
from_live INTEGER NOT NULL DEFAULT 0,
|
|
260
|
+
CONSTRAINT pk_StationSupply PRIMARY KEY (station_id, item_id),
|
|
261
|
+
CONSTRAINT fk_StationSupply_station_id_Station FOREIGN KEY (station_id) REFERENCES Station (station_id) ON DELETE CASCADE,
|
|
262
|
+
CONSTRAINT fk_StationSupply_item_id_Item FOREIGN KEY (item_id) REFERENCES Item (item_id) ON DELETE CASCADE
|
|
263
|
+
) WITHOUT ROWID;
|
|
264
|
+
DELETE FROM StationSupply;
|
|
265
|
+
CREATE INDEX idx_StationSupply_item ON StationSupply (item_id);
|
|
266
|
+
|
|
234
267
|
CREATE VIEW StationBuying AS
|
|
235
268
|
SELECT station_id,
|
|
236
269
|
item_id,
|