tradedangerous 10.16.14__py3-none-any.whl → 10.16.16__py3-none-any.whl

tradedangerous/plugins/eddblink_plug.py

@@ -40,6 +40,34 @@ class DecodingError(PluginException):
     pass
 
 
+def file_line_count(from_file: Path, bufsize: int = 128 * 1024) -> int:
+    """ counts the number of newline characters in a given file. """
+    # Pre-allocate a buffer so we're not putting pressure on the garbage collector,
+    # capture it's counting method so we don't have to keep looking that up on
+    # large files.
+    buf = bytearray(bufsize)
+    counter = buf.count
+
+    total = 0
+
+    with from_file.open("rb") as fh:
+        # Capture the 'readinto' method to avoid lookups.
+        reader = fh.readinto
+
+        # read into the buffer and capture the number of bytes fetched,
+        # which will be 'size' until the last read from the file.
+        read = reader(buf)
+        while read == bufsize:  # nominal case for large files
+            total += counter(b'\n')
+            read = reader(buf)
+
+        # when 0 <= read < bufsize we're on the last page of the
+        # file, so we need to take a slice of the buffer, which creates
+        # a new object and thus we also have to lookup count. it's trivial
+        # but if you have to do it 10,000x it's definitly not a rounding error.
+        return total + buf[:read].count(b'\n')
+
+
 class ImportPlugin(plugins.ImportPluginBase):
     """
     Plugin that downloads data from eddb.
@@ -123,14 +151,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             for result in results:
                 yield result
     
-    @staticmethod
-    def blocks(f, size = 65536):
-        while True:
-            b = f.read(size)
-            if not b:
-                break
-            yield b
-    
+
     def downloadFile(self, path):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
@@ -219,18 +240,15 @@ class ImportPlugin(plugins.ImportPluginBase):
         from_live = 0 if listings_file == self.listingsPath else 1
         
         self.tdenv.DEBUG0(f"Getting total number of entries in {listings_file}...")
-        with open(str(self.dataPath / listings_file), "r", encoding = "utf-8", errors = 'ignore') as f:
-            total += (sum(bl.count("\n") for bl in self.blocks(f)))
+        listings_path = Path(self.dataPath, listings_file)
+        total += file_line_count(listings_path)
         
-        liveList = []
         liveStmt = """UPDATE StationItem
                     SET from_live = 0
                     WHERE station_id = ?"""
         
-        delList = []
         delStmt = "DELETE from StationItem WHERE station_id = ?"
         
-        listingList = []
         listingStmt = """INSERT OR IGNORE INTO StationItem
                         (station_id, item_id, modified,
                          demand_price, demand_units, demand_level,
@@ -238,10 +256,7 @@ class ImportPlugin(plugins.ImportPluginBase):
                         VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )"""
         
         self.tdenv.DEBUG0("Getting list of commodities...")
-        items = []
-        it_result = self.execute("SELECT item_id FROM Item ORDER BY item_id").fetchall()
-        for item in it_result:
-            items.append(item[0])
+        items = [cols[0] for cols in self.execute("SELECT item_id FROM Item ORDER BY item_id")]
         
         self.tdenv.DEBUG0("Getting list of stations...")
         stationList = {
@@ -249,15 +264,17 @@ class ImportPlugin(plugins.ImportPluginBase):
             for (stationID,) in self.execute("SELECT station_id FROM Station")
         }
         
+        stationItems = dict(self.execute('SELECT station_id, UNIXEPOCH(modified) FROM StationItem').fetchall())
+        
         self.tdenv.DEBUG0("Processing entries...")
-        with open(str(self.dataPath / listings_file), "r") as fh:
+        with listings_file.open("r", encoding="utf-8", errors="ignore") as fh:
             prog = pbar.Progress(total, 50)
             listings = csv.DictReader(fh)
             
             cur_station = -1
             
             for listing in listings:
-                if prog.increment(1, postfix = lambda value, goal: f" {(value / total * 100):.0f}% {value} / {total}"):
+                if prog.increment(1, postfix = lambda value, total: f" {(value / total * 100):.0f}% {value} / {total}"):
                     # Do a commit and close the DB every 2%.
                     # This ensures the listings are put in the DB and the WAL is cleared.
                     self.commit()
@@ -272,23 +289,22 @@ class ImportPlugin(plugins.ImportPluginBase):
                     skipStation = False
                     
                     # Check if listing already exists in DB and needs updated.
-                    # Only need to check the date for the first item at a specific station.
-                    result = self.execute("SELECT modified FROM StationItem WHERE station_id = ?", (station_id,)).fetchone()
-                    if result:
-                        updated = timegm(datetime.datetime.strptime(result[0].split('.')[0], '%Y-%m-%d %H:%M:%S').timetuple())
+                    if stationItems.get(station_id):
                         # When the listings.csv data matches the database, update to make from_live == 0.
-                        if int(listing['collected_at']) == updated and not from_live:
+                        if int(listing['collected_at']) == stationItems.get(station_id) and not from_live:
                             self.tdenv.DEBUG1(f"Marking {cur_station} as no longer 'live'.")
                             self.execute(liveStmt, (cur_station,))
                         # Unless the import file data is newer, nothing else needs to be done for this station,
                         # so the rest of the listings for this station can be skipped.
-                        if int(listing['collected_at']) <= updated:
+                        if int(listing['collected_at']) <= stationItems.get(station_id):
                             skipStation = True
                             continue
                         
                         # The data from the import file is newer, so we need to delete the old data for this station.
                         self.tdenv.DEBUG1(f"Deleting old listing data for {cur_station}.")
                         self.execute(delStmt, (cur_station,))
+                        # We've deleted all the items from this station, so remove it.
+                        del stationItems[station_id]
 
                 
                 if skipStation:
@@ -317,7 +333,7 @@ class ImportPlugin(plugins.ImportPluginBase):
             self.tdb.close()
             
             while prog.value < prog.maxValue:
-                prog.increment(1, postfix = lambda value, goal: " " + str(round(value / total * 100)) + "%")
+                prog.increment(1, postfix = lambda value, total: " " + str(round(value / total * 100)) + "%")
             prog.clear()
         
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())

tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '10.16.14'
+__version__ = '10.16.16'
 

{tradedangerous-10.16.14.dist-info → tradedangerous-10.16.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 10.16.14
+Version: 10.16.16
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-10.16.14.dist-info → tradedangerous-10.16.16.dist-info}/RECORD

@@ -21,7 +21,7 @@ tradedangerous/tradeexcept.py,sha256=aZ-Y31MbkjF7lmAzBAbaMsPPE7FEEfuf4gaX2GvriDk
 tradedangerous/tradegui.py,sha256=JbGFnsWupgesk6hrcUgKSdD9NNDyo0U9gh6m3DccAwU,782
 tradedangerous/transfers.py,sha256=NmXXk2aF88YkAvYqc9Syt_aO6d2jJjC-OxoRFoOyQH4,9923
 tradedangerous/utils.py,sha256=PUPvAEqUyxYGqqQa0b_yfLAvq8YVUxK6HfdS-CxM-Lo,5186
-tradedangerous/version.py,sha256=roMp1D3OOt2HhAHHANGah0MJB-QcbbxZ-h7gHE0RKf0,648
+tradedangerous/version.py,sha256=vROLJSJzupfaXUWuLCNyZhDbAbuFvSrk3T-AY9MMAEc,648
 tradedangerous/commands/TEMPLATE.py,sha256=7oXL124aqxGHwnb0h9yRylUiwc6M5QrRrGVrubwI1gg,2124
 tradedangerous/commands/__init__.py,sha256=6B0WuqkFBOll5Hj67yKDAnhmyr5ZAnHc6nzUNEUh384,9640
 tradedangerous/commands/buildcache_cmd.py,sha256=oJvP06fA8svnHrfrpWkHKR16cba8GIhHdMOyZqds18Y,2332
@@ -62,7 +62,7 @@ tradedangerous/misc/progress.py,sha256=-_V7E51sIYUtSxeeA1cphBEW4A_OBuH1guEDxUjik
 tradedangerous/plugins/__init__.py,sha256=zCEVbTem1CAM1cOV9r96H3ikjqza3dd-XoaubE5_xkc,7868
 tradedangerous/plugins/edapi_plug.py,sha256=IQxfDGM9IqwuJbDZRL8RFIyGMWzd0YmeJVWUVPDA3Ik,42275
 tradedangerous/plugins/edcd_plug.py,sha256=ZPtRzLhcQZEiwEo3AoPyk3Uy4UmRLM6gv2Qi1s7K_Vs,14469
-tradedangerous/plugins/eddblink_plug.py,sha256=KpRNjQugXWyA9oFuGzcB2Z0GFqzQRtANig2ksB9NheA,21192
+tradedangerous/plugins/eddblink_plug.py,sha256=3BOX6McE6huXq0AJevbN7m0rZXc_Mns4YnGlaH6nDP4,21964
 tradedangerous/plugins/edmc_batch_plug.py,sha256=3Ptr-SZqaZFR8ViIIrp9Ak7rvfU3zl11AZYBhIceN7s,4224
 tradedangerous/plugins/journal_plug.py,sha256=K1oIeI7E3mb04fvYLXyoAh7fOTyM9NBelibTI88MIDQ,23696
 tradedangerous/plugins/netlog_plug.py,sha256=Gw_HSZWpN17D--OIYEM3Vo8y9SvDOv9UwAUfY24kz28,13460
@@ -71,9 +71,9 @@ tradedangerous/templates/Added.csv,sha256=8o54civQCcS9y7_DBo0GX196XWRbbREQqKDYTK
 tradedangerous/templates/Category.csv,sha256=8xwUDcBZE25T6x6dZGlRUMTCqeDLt3a9LXU5h6hRHV8,250
 tradedangerous/templates/RareItem.csv,sha256=F1RhRnTD82PiwrVUO-ai2ErGH2PTqNnQaDw5mcgljXs,10483
 tradedangerous/templates/TradeDangerous.sql,sha256=1EiJ7cNJQKvdW4X-LQAHw3Y1POc0roKf25LJJy6jGlo,8135
-tradedangerous-10.16.14.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-tradedangerous-10.16.14.dist-info/METADATA,sha256=r0hvIz_n-XPZpEJDyBB0VFb82_Qrz_2-4_3TLFccQco,4442
-tradedangerous-10.16.14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tradedangerous-10.16.14.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
-tradedangerous-10.16.14.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
-tradedangerous-10.16.14.dist-info/RECORD,,
+tradedangerous-10.16.16.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+tradedangerous-10.16.16.dist-info/METADATA,sha256=ZBKc1-gMkTbqzQl9YVoevR_fqnEiTxK6nIIQ8joxx94,4442
+tradedangerous-10.16.16.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tradedangerous-10.16.16.dist-info/entry_points.txt,sha256=pSwa-q0ob443uiKux7xFKYQl8uen66iDTnjdrQhNLx8,92
+tradedangerous-10.16.16.dist-info/top_level.txt,sha256=bF29i-oEltmNICgElEKxNsg83oahJvxg3a7YrxZi9Rk,15
+tradedangerous-10.16.16.dist-info/RECORD,,