tradedangerous 10.16.13__tar.gz → 10.16.15__tar.gz

{tradedangerous-10.16.13 → tradedangerous-10.16.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.16.13
+Version: 10.16.15
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-10.16.13 → tradedangerous-10.16.15}/tradedangerous/plugins/eddblink_plug.py

@@ -91,6 +91,7 @@ class ImportPlugin(plugins.ImportPluginBase):
     def execute(self, sql_cmd, args = None):
         cur = self.tdb.getDB().cursor()
         
+        self.tdenv.DEBUG2(f"SQL-Statement:\n'{sql_cmd},{args}'")
         success = False
         result = None
         while not success:
@@ -109,24 +110,6 @@ class ImportPlugin(plugins.ImportPluginBase):
                     time.sleep(1)
         return result
     
-    def executemany(self, sql_cmd, args):
-        cur = self.tdb.getDB().cursor()
-        
-        success = False
-        result = None
-        while not success:
-            try:
-                result = cur.executemany(sql_cmd, args)
-                success = True
-            except sqlite3.OperationalError as e:
-                if "locked" not in str(e):
-                    success = True
-                    raise sqlite3.OperationalError(e)
-                else:
-                    print("(execute) Database is locked, waiting for access.", end = "\r")
-                    time.sleep(1)
-        return result
-    
     @staticmethod
     def fetchIter(cursor, arraysize = 1000):
         """
@@ -235,13 +218,7 @@ class ImportPlugin(plugins.ImportPluginBase):
         
         from_live = 0 if listings_file == self.listingsPath else 1
         
-        # Used to check if the listings file is using the fdev_id as a temporary
-        # item_id, but the item is in the DB with a permanent item_id.
-        fdev2item = dict()
-        result = self.execute("SELECT fdev_id,item_id FROM Item ORDER BY fdev_id").fetchall()
-        for item in result:
-            fdev2item[item[0]] = item[1]
-        
+        self.tdenv.DEBUG0(f"Getting total number of entries in {listings_file}...")
         with open(str(self.dataPath / listings_file), "r", encoding = "utf-8", errors = 'ignore') as f:
             total += (sum(bl.count("\n") for bl in self.blocks(f)))
         
@@ -260,16 +237,21 @@ class ImportPlugin(plugins.ImportPluginBase):
                          supply_price, supply_units, supply_level, from_live)
                         VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"""
         
+        self.tdenv.DEBUG0("Getting list of commodities...")
         items = []
         it_result = self.execute("SELECT item_id FROM Item ORDER BY item_id").fetchall()
         for item in it_result:
             items.append(item[0])
         
+        self.tdenv.DEBUG0("Getting list of stations...")
         stationList = {
             stationID
             for (stationID,) in self.execute("SELECT station_id FROM Station")
         }
         
+        stationItems = dict(self.execute('SELECT station_id, UNIXEPOCH(modified) FROM StationItem').fetchall())
+        
+        self.tdenv.DEBUG0("Processing entries...")
         with open(str(self.dataPath / listings_file), "r") as fh:
             prog = pbar.Progress(total, 50)
             listings = csv.DictReader(fh)
@@ -277,7 +259,11 @@ class ImportPlugin(plugins.ImportPluginBase):
             cur_station = -1
             
             for listing in listings:
-                prog.increment(1, postfix = lambda value, goal: " " + str(round(value / total * 100)) + "%")
+                if prog.increment(1, postfix = lambda value, goal: f" {(value / total * 100):.0f}% {value} / {total}"):
+                    # Do a commit and close the DB every 2%.
+                    # This ensures the listings are put in the DB and the WAL is cleared.
+                    self.commit()
+                    self.tdb.close()
                 
                 station_id = int(listing['station_id'])
                 if station_id not in stationList:
@@ -288,21 +274,23 @@ class ImportPlugin(plugins.ImportPluginBase):
                     skipStation = False
                     
                     # Check if listing already exists in DB and needs updated.
-                    # Only need to check the date for the first item at a specific station.
-                    result = self.execute("SELECT modified FROM StationItem WHERE station_id = ?", (station_id,)).fetchone()
-                    if result:
-                        updated = timegm(datetime.datetime.strptime(result[0].split('.')[0], '%Y-%m-%d %H:%M:%S').timetuple())
+                    if stationItems.get(station_id):
                         # When the listings.csv data matches the database, update to make from_live == 0.
-                        if int(listing['collected_at']) == updated and not from_live:
-                            liveList.append((cur_station,))
+                        if int(listing['collected_at']) == stationItems.get(station_id) and not from_live:
+                            self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live'.")
+                            self.execute(liveStmt, (cur_station,))
                         # Unless the import file data is newer, nothing else needs to be done for this station,
                         # so the rest of the listings for this station can be skipped.
-                        if int(listing['collected_at']) <= updated:
+                        if int(listing['collected_at']) <= stationItems.get(station_id):
                             skipStation = True
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
-                        delList.append((cur_station,))
+                        self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station}.")
+                        self.execute(delStmt, (cur_station,))
+                        # We've deleted all the items from this station, so remove it.
+                        del stationItems[station_id]
+
                 
                 if skipStation:
                     continue
@@ -320,24 +308,18 @@ class ImportPlugin(plugins.ImportPluginBase):
                 supply_units = int(listing['supply'])
                 supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1
                 
-                listingList.append((station_id, item_id, modified,
+                self.tdenv.DEBUG1(f"Inserting new listing data for {station_id}.")
+                self.execute(listingStmt, (station_id, item_id, modified,
                                     demand_price, demand_units, demand_level,
                                     supply_price, supply_units, supply_level, from_live))
             
+            # Do a final commit to be sure
+            self.commit()
+            self.tdb.close()
+            
             while prog.value < prog.maxValue:
                 prog.increment(1, postfix = lambda value, goal: " " + str(round(value / total * 100)) + "%")
             prog.clear()
-            
-            self.tdenv.NOTE("Import file processing complete, updating database. {}", self.now())
-            if liveList:
-                self.tdenv.NOTE("Marking data now in the EDDB listings.csv as no longer 'live'. {}", self.now())
-                self.executemany(liveStmt, liveList)
-            if delList:
-                self.tdenv.NOTE("Deleting old listing data. {}", self.now())
-                self.executemany(delStmt, delList)
-            if listingList:
-                self.tdenv.NOTE("Inserting new listing data. {}", self.now())
-                self.executemany(listingStmt, listingList)
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
     
@@ -497,21 +479,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         # Remake the .db files with the updated info.
         if buildCache:
             self.tdb.close()
-            cache.buildCache(self.tdb, self.tdenv)
+            self.tdb.reloadCache()
         
         self.tdenv.ignoreUnknown = True
         
-        success = False
-        while not success:
-            try:
-                self.tdenv.DEBUG0("Loading Database. {}", self.now())
-                self.tdb.load(maxSystemLinkLy = self.tdenv.maxSystemLinkLy)
-                success = True
-            except sqlite3.OperationalError:
-                print("Database is locked, waiting for access.", end = "\r")
-                time.sleep(1)
-        self.tdenv.DEBUG0("Database loaded.")
-        
         if self.getOption("purge"):
             self.purgeSystems()
             # self.commit()
@@ -522,9 +493,6 @@ class ImportPlugin(plugins.ImportPluginBase):
             if self.downloadFile(self.liveListingsPath) or self.getOption("force"):
                 self.importListings(self.liveListingsPath)
         
-        # self.commit()
-        self.tdb.close()
-        
         if self.getOption("listings"):
             self.tdenv.NOTE("Regenerating .prices file.")
             cache.regeneratePricesFile(self.tdb, self.tdenv)

{tradedangerous-10.16.13 → tradedangerous-10.16.15}/tradedangerous/plugins/spansh_plug.py

@@ -90,7 +90,6 @@ class ImportPlugin(plugins.ImportPluginBase):
             if rib_path.exists():
                 rib_path.rename(ri_path)
         
-        # self.known_space = self.load_known_space()
         self.known_systems = self.load_known_systems()
         self.known_stations = self.load_known_stations()
         self.known_commodities = self.load_known_commodities()
@@ -99,34 +98,13 @@ class ImportPlugin(plugins.ImportPluginBase):
         return self.tdenv.uprint(*args, **kwargs)
 
     def run(self):
-        # fs.ensurefolder(self.tdenv.tmpDir)
-        # filePath = self.tdenv.tmpDir / Path("spansh.prices")
         if not self.tdenv.detail:
             self.print('This will take at least several minutes...')
             self.print('You can increase verbosity (-v) to get a sense of progress')
         with Timing() as timing:
-        # with open(filePath, 'w') as f, Timing() as timing:
-        #     self.print(f'Writing prices to {filePath}')
-        #     f.write('# Generated from spansh galaxy data\n')
-        #     f.write(f'# Source: {self.file or self.url}\n')
-        #     f.write('#\n')
-        #     f.write((
-        #         '#     {name:50s}  {sell:>7s}  {buy:>7s}  '
-        #         '{demand:>11s}  {supply:>11s}  {ts}\n'
-        #     ).format(
-        #         name='Item Name',
-        #         sell='SellCr',
-        #         buy='BuyCr',
-        #         demand='Demand',
-        #         supply='Supply',
-        #         ts='Timestamp',
-        #     ))
             system_count = 0
             total_station_count = 0
             total_commodity_count = 0
-            # self.need_commit = False
-            # self.update_cache = False
-            # seen_stations = set()
             for system, stations in self.data_stream():
                 self.ensure_system(system)
                 station_count = 0
@@ -135,16 +113,9 @@ class ImportPlugin(plugins.ImportPluginBase):
                     fq_station_name = f'@{system.name.upper()}/{station.name}'
                     if self.maxage and (datetime.now() - station.modified) > timedelta(days=self.maxage):
                         if self.tdenv.detail:
-                            self.print(f'        |  {fq_station_name:50s}  |  Skipping station due to age: {datetime.now() - station.modified}, ts: {station.modified}')
+                            self.print(f'        |  {fq_station_name:50s}  |  Skipping station due to age: {(datetime.now() - station.modified) / timedelta (days=1):.2f} days old')
                         continue
-                    # if (system.name.upper(), station.name.upper()) in seen_stations:
-                    #     if self.tdenv.detail:
-                    #         self.print(f'        |  {fq_station_name:50s}  |  Skipping duplicate station record')
-                    #     continue
-                    # seen_stations.add((system.name.upper(), station.name.upper()))
                     self.ensure_station(system, station)
-                    # f.write('\n')
-                    # f.write(f'@ {system.name.upper()}/{station.name}\n')
                     
                     items = []
                     for commodity in commodities:
@@ -176,23 +147,6 @@ class ImportPlugin(plugins.ImportPluginBase):
                             commodity_count += 1
                         self.execute('COMMIT')
                     
-                    # categories = self.categorise_commodities(commodities)
-                    # for category_name, category_commodities in categories.items():
-                    #     f.write(f'   + {category_name}\n')
-                    #     for commodity in category_commodities:
-                    #         commodity = self.ensure_commodity(commodity)
-                    #         f.write((
-                    #             '      {name:50s}  {sell:7d}  {buy:7d}  '
-                    #             '{demand:10d}?  {supply:10d}?  {modified}\n'
-                    #         ).format(
-                    #             name=commodity.name,
-                    #             sell=commodity.sell,
-                    #             buy=commodity.buy,
-                    #             demand=commodity.demand,
-                    #             supply=commodity.supply,
-                    #             modified=commodity.modified,
-                    #         ))
-                    #         commodity_count += 1
                     if commodity_count:
                         station_count += 1
                 if station_count:
@@ -204,16 +158,6 @@ class ImportPlugin(plugins.ImportPluginBase):
                             f'{system_count:6d}  |  {system.name.upper():50s}  |  '
                             f'{station_count:3d} st  {commodity_count:6d} co'
                         )
-                # self.execute('COMMIT')
-                # if self.need_commit:
-                #     self.execute('COMMIT')
-                #     self.need_commit = False
-                #     self.update_cache = True
-                    
-            # Need to make sure cached tables are updated, if changes were made
-            # if self.update_cache:
-            #     for table in [ "Item", "Station", "System" ]:
-            #         _, path = csvexport.exportTableToFile( self.tdb, self.tdenv, table )
             
             self.execute('COMMIT')
             self.tdb.close()
@@ -230,13 +174,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             self.print('Exporting to cache...')
             cache.regeneratePricesFile(self.tdb, self.tdenv)
             self.print(f'Cache export completed in {timedelta(seconds=int(timing.elapsed))!s}')
-
-        # if not self.listener:
-        #     with Timing() as timing:
-        #         self.print('Importing to database...')
-        #         self.tdenv.mergeImport = True
-        #         cache.importDataFromFile(self.tdb, self.tdenv, filePath)
-        #         self.print(f'Database import completed in {timedelta(seconds=int(timing.elapsed))!s}')
+        
         return False
 
     def data_stream(self):
@@ -253,11 +191,6 @@ class ImportPlugin(plugins.ImportPluginBase):
         elif self.file:
             self.print(f'Reading prices from local file: {self.file}')
             stream = open(self.file, 'r', encoding='utf8')
-        # else:
-        #     url = self.url or SOURCE_URL
-        #     self.print(f'Reading prices from remote URL: {url}')
-        #     req = requests.get(url, stream=True)
-        #     stream = req.iter_lines(decode_unicode=True)
         return ingest_stream(stream)
 
     def categorise_commodities(self, commodities):
@@ -280,20 +213,6 @@ class ImportPlugin(plugins.ImportPluginBase):
                 attempts -= 1
                 self.print(f'Retrying query \'{query}\': {ex!s}')
                 time.sleep(1)
-
-    # def load_known_space(self):
-    #     cache = {}
-    #     result = self.execute(
-    #         '''
-    #         SELECT System.name, Station.name FROM System
-    #         LEFT JOIN Station USING (system_id)
-    #         '''
-    #     ).fetchall()
-    #     for system, station in result:
-    #         cache.setdefault(system.upper(), set())
-    #         if station is not None:
-    #             cache[system.upper()].add(station.upper())
-    #     return cache
     
     def load_known_systems(self):
         try:
@@ -382,19 +301,6 @@ class ImportPlugin(plugins.ImportPluginBase):
 
     def ensure_commodity(self, commodity):
         if commodity.id in self.known_commodities:
-            # if self.known_commodities[commodity.id] != commodity.name:
-            #     if self.tdenv.detail >= 3:
-            #         self.print(f'        |    -  {commodity.name:45s}  |  Replace with pre-existing "{self.known_commodities[commodity.id]}"')
-            #     return Commodity(
-            #         id=commodity.id,
-            #         name=self.known_commodities[commodity.id],
-            #         category=commodity.category,
-            #         demand=commodity.demand,
-            #         supply=commodity.supply,
-            #         sell=commodity.sell,
-            #         buy=commodity.buy,
-            #         modified=commodity.modified,
-            #     )
             return commodity
         self.execute(
             '''

{tradedangerous-10.16.13 → tradedangerous-10.16.15}/tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '10.16.13'
+__version__ = '10.16.15'
 

{tradedangerous-10.16.13 → tradedangerous-10.16.15}/tradedangerous.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.16.13
+Version: 10.16.15
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus