tradedangerous 10.14.3__tar.gz → 10.15.0__tar.gz

{tradedangerous-10.14.3 → tradedangerous-10.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.14.3
+Version: 10.15.0
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-10.14.3 → tradedangerous-10.15.0}/tradedangerous/cache.py

@@ -632,17 +632,34 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals
     removedItems = len(zeros)
     
     if items:
-        db.executemany("""
-            INSERT OR REPLACE INTO StationItem (
-                station_id, item_id, modified,
-                demand_price, demand_units, demand_level,
-                supply_price, supply_units, supply_level
-            ) VALUES (
-                ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
-                ?, ?, ?,
-                ?, ?, ?
-            )
-        """, items)
+        for item in items:
+            try:
+                db.execute("""
+                    INSERT OR REPLACE INTO StationItem (
+                        station_id, item_id, modified,
+                        demand_price, demand_units, demand_level,
+                        supply_price, supply_units, supply_level
+                    ) VALUES (
+                        ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
+                        ?, ?, ?,
+                        ?, ?, ?
+                    )
+                """, item)
+            except sqlite3.IntegrityError as e:
+                print(e)
+                print(item)
+                raise e
+        # db.executemany("""
+        #     INSERT OR REPLACE INTO StationItem (
+        #         station_id, item_id, modified,
+        #         demand_price, demand_units, demand_level,
+        #         supply_price, supply_units, supply_level
+        #     ) VALUES (
+        #         ?, ?, IFNULL(?, CURRENT_TIMESTAMP),
+        #         ?, ?, ?,
+        #         ?, ?, ?
+        #     )
+        # """, items)
     updatedItems = len(items)
     
     tdenv.DEBUG0("Marking populated stations as having a market")
@@ -654,6 +671,7 @@ def processPricesFile(tdenv, db, pricesPath, pricesFh = None, defaultZero = Fals
              ")"
     )
     
+    tdenv.DEBUG0(f'Committing...')
     db.commit()
     
     changes = " and ".join("{} {}".format(v, k) for k, v in {

{tradedangerous-10.14.3 → tradedangerous-10.15.0}/tradedangerous/plugins/spansh_plug.py

@@ -9,7 +9,7 @@ import requests
 import simdjson
 import sqlite3
 
-from .. import plugins, cache, fs
+from .. import plugins, cache, fs, transfers
 
 SOURCE_URL = 'https://downloads.spansh.co.uk/galaxy_stations.json'
 
@@ -55,6 +55,7 @@ class ImportPlugin(plugins.ImportPluginBase):
     pluginOptions = {
         'url':  f'URL to download galaxy data from (defaults to {SOURCE_URL})',
         'file': 'Local filename to import galaxy data from; use "-" to load from stdin',
+        'maxage': 'Skip all entries older than specified age in days, ex.: maxage=1.5',
         'listener': 'For use by TD-listener, prevents updating cache from generated prices file',
     }
 
@@ -62,6 +63,7 @@ class ImportPlugin(plugins.ImportPluginBase):
         super().__init__(*args, **kwargs)
         self.url = self.getOption('url')
         self.file = self.getOption('file')
+        self.maxage = float(self.getOption('maxage'))
         self.listener = self.getOption('listener')
         assert not (self.url and self.file), 'Provide either url or file, not both'
         if self.file and (self.file != '-'):
@@ -104,6 +106,10 @@ class ImportPlugin(plugins.ImportPluginBase):
                 station_count = 0
                 commodity_count = 0
                 for station, commodities in stations:
+                    if (datetime.now() - station.modified) > timedelta(days=self.maxage):
+                        if self.tdenv.detail >= 1:
+                            self.print(f'        |  @{system.name.upper()}/{station.name.upper():50s}  |  Skipping station due to age: {datetime.now() - station.modified}, ts: {station.modified}')
+                        continue
                     if (system.name.upper(), station.name.upper()) in seen_stations:
                         fq_station_name = f'@{system.name.upper()}/{station.name}'
                         if self.tdenv.detail >= 1:
@@ -156,17 +162,24 @@ class ImportPlugin(plugins.ImportPluginBase):
         return False
 
     def data_stream(self):
+        if not self.file:
+            url = self.url or SOURCE_URL
+            self.print(f'Downloading prices from remote URL: {url}')
+            self.file = self.tdenv.tmpDir / Path("galaxy_stations.json")
+            transfers.download(self.tdenv, url, self.file)
+            self.print(f'Download complete, saved to local file: {self.file}')
+
         if self.file == '-':
             self.print('Reading prices from stdin')
             stream = sys.stdin
         elif self.file:
             self.print(f'Reading prices from local file: {self.file}')
             stream = open(self.file, 'r', encoding='utf8')
-        else:
-            url = self.url or SOURCE_URL
-            self.print(f'Reading prices from remote URL: {url}')
-            req = requests.get(url, stream=True)
-            stream = req.iter_lines(decode_unicode=True)
+        # else:
+        #     url = self.url or SOURCE_URL
+        #     self.print(f'Reading prices from remote URL: {url}')
+        #     req = requests.get(url, stream=True)
+        #     stream = req.iter_lines(decode_unicode=True)
         return ingest_stream(stream)
 
     def categorise_commodities(self, commodities):
@@ -176,15 +189,15 @@ class ImportPlugin(plugins.ImportPluginBase):
         return categories
 
     def execute(self, query, *params, **kwparams):
-        attempts = 5
+        # attempts = 5
         cursor = self.tdb.getDB().cursor()
         while True:
             try:
                 return cursor.execute(query, params or kwparams)
             except sqlite3.OperationalError as ex:
-                if not attempts:
-                    raise
-                attempts -= 1
+                # if not attempts:
+                #     raise
+                # attempts -= 1
                 self.print(f'Retrying query: {ex!s}')
                 time.sleep(1)
 

{tradedangerous-10.14.3 → tradedangerous-10.15.0}/tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '10.14.3'
+__version__ = '10.15.0'
 

{tradedangerous-10.14.3 → tradedangerous-10.15.0}/tradedangerous.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.14.3
+Version: 10.15.0
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus