tradedangerous 10.13.10__py3-none-any.whl → 10.14.2__py3-none-any.whl

tradedangerous/cache.py

@@ -285,12 +285,12 @@ def getStationByNameIndex(cur):
     """ Build station index in STAR/Station notation """
     cur.execute("""
             SELECT station_id,
-                    UPPER(system.name) || '/' || UPPER(station.name)
+                    system.name || '/' || station.name
               FROM System
                    INNER JOIN Station
                       USING (system_id)
         """)
-    return { name: ID for (ID, name) in cur }
+    return { name.upper(): ID for (ID, name) in cur }
 
 
 def getItemByNameIndex(cur):

tradedangerous/plugins/spansh_plug.py

@@ -0,0 +1,379 @@
+import os
+import sys
+import time
+from datetime import datetime, timedelta
+from collections import namedtuple
+from pathlib import Path
+
+import requests
+import simdjson
+import sqlite3
+
+from .. import plugins
+
+SOURCE_URL = 'https://downloads.spansh.co.uk/galaxy_stations.json'
+
+STATION_TYPE_MAP = {
+    'Drake-Class Carrier': 24,  # fleet carriers
+    'Settlement': 25,           # odyssey settlements
+}
+
+System = namedtuple('System', 'name,pos_x,pos_y,pos_z,modified')
+Station = namedtuple('Station', 'name,distance,max_pad_size,market,black_market,shipyard,outfitting,rearm,refuel,repair,planetary,type,modified')
+Commodity = namedtuple('Commodity', 'id,name,category,demand,supply,sell,buy,modified')
+
+
+class Timing:
+
+    def __init__(self):
+        self.start_ts = None
+        self.end_ts = None
+
+    def __enter__(self):
+        self.start_ts = time.perf_counter()
+        self.end_ts = None
+        return self
+
+    def __exit__(self, *args):
+        self.end_ts = time.perf_counter()
+
+    @property
+    def elapsed(self):
+        if self.start_ts is None:
+            return None
+        return (self.end_ts or time.perf_counter()) - self.start_ts
+
+    @property
+    def is_finished(self):
+        return self.end_ts is not None
+
+
+class ImportPlugin(plugins.ImportPluginBase):
+    """Plugin that downloads data from https://spansh.co.uk/dumps.
+    """
+
+    pluginOptions = {
+        'url':  f'URL to download galaxy data from (defaults to {SOURCE_URL})',
+        'file': 'Local filename to import galaxy data from; use "-" to load from stdin',
+    }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.url = self.getOption('url')
+        self.file = self.getOption('file')
+        assert not (self.url and self.file), 'Provide either url or file, not both'
+        if self.file and (self.file != '-'):
+            self.file = (Path(self.tdenv.cwDir) / self.file).resolve()
+        self.known_space = self.load_known_space()
+        self.known_commodities = self.load_known_commodities()
+
+    def print(self, *args, **kwargs):
+        return self.tdenv.uprint(*args, **kwargs)
+
+    def run(self):
+        self.tdenv.filename = str(self.tdb.pricesPath)
+        if self.tdenv.detail < 1:
+            self.print('This will take at least several minutes...')
+            self.print('You can increase verbosity (-v) to get a sense of progress')
+        with open(self.tdb.pricesPath, 'w') as f, Timing() as timing:
+            self.print(f'Writing prices to {self.tdb.pricesPath}')
+            f.write('# Generated from spansh galaxy data\n')
+            f.write(f'# Source: {self.file or self.url}\n')
+            f.write('#\n')
+            f.write((
+                '#     {name:50s}  {sell:>7s}  {buy:>7s}  '
+                '{demand:>11s}  {supply:>11s}  {ts}\n'
+            ).format(
+                name='Item Name',
+                sell='SellCr',
+                buy='BuyCr',
+                demand='Demand',
+                supply='Supply',
+                ts='Timestamp',
+            ))
+            system_count = 0
+            total_station_count = 0
+            total_commodity_count = 0
+            self.need_commit = False
+            seen_stations = set()
+            for system, stations in self.data_stream():
+                self.ensure_system(system)
+                station_count = 0
+                commodity_count = 0
+                for station, commodities in stations:
+                    if (system.name.upper(), station.name.upper()) in seen_stations:
+                        fq_station_name = f'@{system.name.upper()}/{station.name}'
+                        if self.tdenv.detail >= 1:
+                            self.print(f'        |  {fq_station_name:50s}  |  Skipping duplicate station record')
+                        continue
+                    seen_stations.add((system.name.upper(), station.name.upper()))
+                    self.ensure_station(system, station)
+                    f.write('\n')
+                    f.write(f'@ {system.name.upper()}/{station.name}\n')
+                    categories = self.categorise_commodities(commodities)
+                    for category_name, category_commodities in categories.items():
+                        f.write(f'   + {category_name}\n')
+                        for commodity in category_commodities:
+                            commodity = self.ensure_commodity(station, commodity)
+                            f.write((
+                                '      {name:50s}  {sell:7d}  {buy:7d}  '
+                                '{demand:10d}?  {supply:10d}?  {modified}\n'
+                            ).format(
+                                name=commodity.name,
+                                sell=commodity.sell,
+                                buy=commodity.buy,
+                                demand=commodity.demand,
+                                supply=commodity.supply,
+                                modified=commodity.modified,
+                            ))
+                            commodity_count += 1
+                    station_count += 1
+                system_count += 1
+                total_station_count += station_count
+                total_commodity_count += commodity_count
+                if self.tdenv.detail >= 1:
+                    self.print(
+                        f'{system_count:6d}  |  {system.name.upper():50s}  |  '
+                        f'{station_count:3d} st  {commodity_count:6d} co'
+                    )
+                if self.need_commit:
+                    self.execute('COMMIT')
+                    self.need_commit = False
+            self.print(
+                f'{timedelta(seconds=int(timing.elapsed))!s}  Done  '
+                f'{total_station_count} st  {total_commodity_count} co'
+            )
+        return True
+
+    def data_stream(self):
+        if self.file == '-':
+            self.print('Reading prices from stdin')
+            stream = sys.stdin
+        elif self.file:
+            self.print(f'Reading prices from local file: {self.file}')
+            stream = open(self.file, 'r', encoding='utf8')
+        else:
+            url = self.url or SOURCE_URL
+            self.print(f'Reading prices from remote URL: {url}')
+            req = requests.get(url, stream=True)
+            stream = req.iter_lines(decode_unicode=True)
+        return ingest_stream(stream)
+
+    def categorise_commodities(self, commodities):
+        categories = {}
+        for commodity in commodities:
+            categories.setdefault(commodity.category, []).append(commodity)
+        return categories
+
+    def execute(self, query, *params, **kwparams):
+        attempts = 5
+        cursor = self.tdb.getDB().cursor()
+        while True:
+            try:
+                return cursor.execute(query, params or kwparams)
+            except sqlite3.OperationalError as ex:
+                if not attempts:
+                    raise
+                attempts -= 1
+                self.print(f'Retrying query: {ex!s}')
+                time.sleep(1)
+
+    def load_known_space(self):
+        cache = {}
+        result = self.execute(
+            '''
+            SELECT System.name, Station.name FROM System
+            LEFT JOIN Station USING (system_id)
+            '''
+        ).fetchall()
+        for system, station in result:
+            cache.setdefault(system.upper(), set())
+            if station is not None:
+                cache[system.upper()].add(station.upper())
+        return cache
+
+    def load_known_commodities(self):
+        return dict(self.execute('SELECT fdev_id, name FROM Item').fetchall())
+
+    def ensure_system(self, system):
+        if system.name.upper() in self.known_space:
+            return
+        self.execute(
+            '''
+            INSERT INTO System (name, pos_x, pos_y, pos_z, modified) VALUES (?, ?, ?, ?, ?)
+            ''',
+            system.name, system.pos_x, system.pos_y, system.pos_z, system.modified,
+        )
+        self.need_commit = True
+        if self.tdenv.detail >= 2:
+            self.print(f'        |  {system.name.upper():50s}  |  Added missing system')
+        self.known_space[system.name.upper()] = set()
+
+    def ensure_station(self, system, station):
+        if station.name.upper() in self.known_space.get(system.name.upper(), set()):
+            return
+        self.execute(
+            '''
+            INSERT INTO Station (
+                system_id,
+                name,
+                ls_from_star,
+                max_pad_size,
+                market,
+                blackmarket,
+                shipyard,
+                outfitting,
+                rearm,
+                refuel,
+                repair,
+                planetary,
+                modified,
+                type_id
+            )
+            VALUES (
+                (SELECT system_id FROM System WHERE upper(name) = ?),
+                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+            )
+            ''',
+            system.name.upper(),
+            station.name,
+            station.distance,
+            station.max_pad_size,
+            self.bool_yn(station.market),
+            self.bool_yn(station.black_market),
+            self.bool_yn(station.shipyard),
+            self.bool_yn(station.outfitting),
+            self.bool_yn(station.rearm),
+            self.bool_yn(station.refuel),
+            self.bool_yn(station.repair),
+            self.bool_yn(station.planetary),
+            station.modified,
+            station.type,
+        )
+        self.need_commit = True
+        if self.tdenv.detail >= 2:
+            self.print(f'        |  {station.name:50s}  |  Added missing station')
+        self.known_space[system.name.upper()].add(station.name.upper())
+
+    def ensure_commodity(self, station, commodity):
+        if commodity.id in self.known_commodities:
+            if self.known_commodities[commodity.id] != commodity.name:
+                if self.tdenv.detail >= 3:
+                    self.print(f'        |    -  {station.name:45s}  |  Replace "{commodity.name}" with pre-existing "{self.known_commodities[commodity.id]}"')
+                return Commodity(
+                    id=commodity.id,
+                    name=self.known_commodities[commodity.id],
+                    category=commodity.category,
+                    demand=commodity.demand,
+                    supply=commodity.supply,
+                    sell=commodity.sell,
+                    buy=commodity.buy,
+                    modified=commodity.modified,
+                )
+            return commodity
+        self.execute(
+            '''
+            INSERT INTO Item (category_id, name, fdev_id)
+            VALUES ((SELECT category_id FROM Category WHERE upper(name) = ?), ?, ?)
+            ''',
+            commodity.category.upper(),
+            commodity.name,
+            commodity.id,
+        )
+        self.need_commit = True
+        if self.tdenv.detail >= 2:
+            self.print(f'        |  {commodity.name:50s}  |  Added missing commodity')
+        self.known_commodities[commodity.id] = commodity.name
+        return commodity
+
+    def bool_yn(self, value):
+        return '?' if value is None else ('Y' if value else 'N')
+
+
+def ingest_stream(stream):
+    """Ingest a spansh-style galaxy dump, yielding system-level data."""
+    line = next(stream)
+    assert line.rstrip(' \n,') == '['
+    for line in stream:
+        line = line.rstrip().rstrip(',')
+        if line == ']':
+            break
+        system_data = simdjson.Parser().parse(line)
+        coords = system_data.get('coords', {})
+        yield (
+            System(
+                name=system_data.get('name', 'Unnamed'),
+                pos_x=coords.get('x', 999999),
+                pos_y=coords.get('y', 999999),
+                pos_z=coords.get('z', 999999),
+                modified=parse_ts(system_data.get('date')),
+            ),
+            ingest_stations(system_data),
+        )
+
+
+def ingest_stations(system_data):
+    """Ingest system-level data, yielding station-level data."""
+    targets = [system_data, *system_data.get('bodies', ())]
+    is_planetary = False
+    for target in targets:
+        for station_data in target.get('stations', ()):
+            services = set(station_data.get('services', ()))
+            if 'Market' not in services:
+                continue
+            market = station_data.get('market', {})
+            if not market.get('commodities'):
+                continue
+            landing_pads = station_data.get('landingPads', {})
+            max_pad_size = '?'
+            if landing_pads.get('large'):
+                max_pad_size = 'L'
+            elif landing_pads.get('medium'):
+                max_pad_size = 'M'
+            elif landing_pads.get('small'):
+                max_pad_size = 'S'
+            yield (
+                Station(
+                    name=station_data.get('name', 'Unnamed'),
+                    distance=station_data.get('distanceToArrival', 999999),
+                    max_pad_size=max_pad_size,
+                    market=True,
+                    black_market='Black Market' in services,
+                    shipyard='Shipyard' in services,
+                    outfitting='Outfitting' in services,
+                    rearm='Restock' in services,
+                    refuel='Refuel' in services,
+                    repair='Repair' in services,
+                    planetary=is_planetary,
+                    type=STATION_TYPE_MAP.get(station_data.get('type'), 0),
+                    modified=parse_ts(station_data.get('updateTime')),
+                ),
+                ingest_market(market),
+            )
+        # first target is system stations, everything else is planetary
+        is_planetary = True
+
+
+def ingest_market(market):
+    """Ingest station-level market data, yielding commodities."""
+    for commodity in market['commodities']:
+        yield Commodity(
+            id=commodity.get('commodityId'),
+            name=commodity.get('name', 'Unnamed'),
+            category=commodity.get('category', 'Uncategorised'),
+            demand=commodity.get('demand', 0),
+            supply=commodity.get('supply', 0),
+            sell=commodity.get('sellPrice', 0),
+            buy=commodity.get('buyPrice', 0),
+            modified=parse_ts(market.get('updateTime'))
+        )
+
+
+def parse_ts(ts):
+    if ts is None:
+        return None
+    if ts.endswith('+00'):
+        ts = ts[:-3]
+    if '.' not in ts:
+        ts += '.0'
+    return datetime.strptime(ts, '%Y-%m-%d %H:%M:%S.%f').replace(microsecond=0)

tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '10.13.10'
+__version__ = '10.14.2'
 

{tradedangerous-10.13.10.dist-info → tradedangerous-10.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.13.10
+Version: 10.14.2
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus
@@ -21,6 +21,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: appJar
+Requires-Dist: pysimdjson
 
 
 ----------

{tradedangerous-10.13.10.dist-info → tradedangerous-10.14.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 tradedangerous/__init__.py,sha256=5ZeypoZaM8hlh6c-yTkD8x5hZYP8q3Q8a3bVeicHr90,1122
-tradedangerous/cache.py,sha256=_kcGrEI1CMbMftJvGU3QpXIUPwEj_2-BcL-WHmGGHWA,32948
+tradedangerous/cache.py,sha256=t9--q6etNH5vigvA_1VLf_XTdoC_116qkjnqswhshpg,32942
 tradedangerous/cli.py,sha256=ycGkzMszwpP9OrngFIyhQbYoZ2wzRhJMTDCW1YhLrNI,4727
 tradedangerous/corrections.py,sha256=QMs-7MKLw2imFgIHthnwcpqWT1yJTb3CrABJw9LaKLA,1441
 tradedangerous/csvexport.py,sha256=OcOKe_VIgafw6rCvG3k5tM11KwwoYXMNY5DSSfCC0mU,8705
@@ -21,7 +21,7 @@ tradedangerous/tradeexcept.py,sha256=aZ-Y31MbkjF7lmAzBAbaMsPPE7FEEfuf4gaX2GvriDk
 tradedangerous/tradegui.py,sha256=JbGFnsWupgesk6hrcUgKSdD9NNDyo0U9gh6m3DccAwU,782
 tradedangerous/transfers.py,sha256=NmXXk2aF88YkAvYqc9Syt_aO6d2jJjC-OxoRFoOyQH4,9923
 tradedangerous/utils.py,sha256=PUPvAEqUyxYGqqQa0b_yfLAvq8YVUxK6HfdS-CxM-Lo,5186
-tradedangerous/version.py,sha256=XS35iBSFdbTM6DjV7JPdxmKUR629HgyX_cKgx8qIsaM,648
+tradedangerous/version.py,sha256=5iuvqnu0uphkiczIVtwP6oY8olhV7oHQ5oaAUqtQt6g,647
 tradedangerous/commands/TEMPLATE.py,sha256=7oXL124aqxGHwnb0h9yRylUiwc6M5QrRrGVrubwI1gg,2124
 tradedangerous/commands/__init__.py,sha256=6B0WuqkFBOll5Hj67yKDAnhmyr5ZAnHc6nzUNEUh384,9640
 tradedangerous/commands/buildcache_cmd.py,sha256=oJvP06fA8svnHrfrpWkHKR16cba8GIhHdMOyZqds18Y,2332
@@ -66,13 +66,14 @@ tradedangerous/plugins/eddblink_plug.py,sha256=n8AzLh2pJdHdar1SnqSSjCwWPl4f-OKQ8
 tradedangerous/plugins/edmc_batch_plug.py,sha256=3Ptr-SZqaZFR8ViIIrp9Ak7rvfU3zl11AZYBhIceN7s,4224
 tradedangerous/plugins/journal_plug.py,sha256=K1oIeI7E3mb04fvYLXyoAh7fOTyM9NBelibTI88MIDQ,23696
 tradedangerous/plugins/netlog_plug.py,sha256=Gw_HSZWpN17D--OIYEM3Vo8y9SvDOv9UwAUfY24kz28,13460
+tradedangerous/plugins/spansh_plug.py,sha256=ImU1tp995LhzFX7sJXApLiwrxx29b2WXR0jtWvm5O0U,14485
 tradedangerous/templates/Added.csv,sha256=8o54civQCcS9y7_DBo0GX196XWRbbREQqKDYTKibsgQ,649
 tradedangerous/templates/DefaultShipIndex.json,sha256=m5cI3vkKiqRk1VKO1Z_8LZrG9nczV0PUMDfBSt4-1RM,94739
 tradedangerous/templates/RareItem.csv,sha256=F1RhRnTD82PiwrVUO-ai2ErGH2PTqNnQaDw5mcgljXs,10483
 tradedangerous/templates/TradeDangerous.sql,sha256=6sjEogGHy-9zYpjPo0Y2a5tElowmHFyJNwrimOUBfHk,8079
-tradedangerous-10.13.10.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-tradedangerous-10.13.10.dist-info/METADATA,sha256=Aq7LfqX_C0zTErgC2jOcfVSem8ZJJhZ1vBW0q9ajo1M,4396
-tradedangerous-10.13.10.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-tradedangerous-10.13.10.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
-tradedangerous-10.13.10.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
-tradedangerous-10.13.10.dist-info/RECORD,,
+tradedangerous-10.14.2.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+tradedangerous-10.14.2.dist-info/METADATA,sha256=F90OuVRZ5eI8SzlzQDdmhrrOQQbEa0RnAMlRKXX-KBk,4421
+tradedangerous-10.14.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tradedangerous-10.14.2.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
+tradedangerous-10.14.2.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
+tradedangerous-10.14.2.dist-info/RECORD,,

{tradedangerous-10.13.10.dist-info → tradedangerous-10.14.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.38.4)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any