tradedangerous 12.0.5__py3-none-any.whl → 12.0.6__py3-none-any.whl

tradedangerous/plugins/eddblink_plug.py

@@ -1,10 +1,8 @@
-from __future__ import annotations
-
 """
 Import plugin that uses data files from 
 https://elite.tromador.com/ to update the Database.
 """
-
+from __future__ import annotations
 
 from email.utils import parsedate_to_datetime
 from pathlib import Path
@@ -40,7 +38,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
     if not listings.exists():
         tdenv.NOTE("File not found, aborting: {}", listings)
         return 0
-
+    
     tdenv.DEBUG0(f"Getting total number of entries in {listings}...")
     count = file_line_count(listings)
     if count <= 1:
@@ -49,7 +47,7 @@ def _count_listing_entries(tdenv: TradeEnv, listings: Path) -> int:
         else:
             tdenv.DEBUG0("Listings file is empty, nothing to do.")
         return 0
-
+    
     return count + 1  # kfsone: Doesn't the header already make this + 1?
 
 
@@ -108,10 +106,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         'optimize':     "Optimize ('vacuum') database after processing.",
         'solo':         "Don't download crowd-sourced market data. (Implies '-O skipvend', supercedes '-O all', '-O clean', '-O listings'.)",
     }
-
+    
     def __init__(self, tdb, tdenv):
         super().__init__(tdb, tdenv)
-
+        
         self.dataPath = os.environ.get('TD_EDDB') or self.tdenv.tmpDir
         self.categoriesPath = Path("Category.csv")
         self.commoditiesPath = Path("Item.csv")
@@ -129,10 +127,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         self.listingsPath = Path("listings.csv")
         self.liveListingsPath = Path("listings-live.csv")
         self.pricesPath = Path("listings.prices")
-
+    
     def now(self):
         return datetime.datetime.now()
-
+    
     def downloadFile(self, path):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
@@ -141,9 +139,9 @@ class ImportPlugin(plugins.ImportPluginBase):
             localPath = Path(self.tdb.dataPath, path)
         else:
             localPath = Path(self.dataPath, path)
-
+        
         url = BASE_URL + str(path)
-
+        
         self.tdenv.NOTE("Checking for update to '{}'.", path)
         # Use an HTTP Request header to obtain the Last-Modified and Content-Length headers.
         # Also, tell the server to give us the un-compressed length of the file by saying
@@ -154,48 +152,48 @@ class ImportPlugin(plugins.ImportPluginBase):
         except Exception as e:  # pylint: disable=broad-exception-caught
             self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", url, str(e))
             return False
-
+        
         last_modified = response.headers.get("last-modified")
         dump_mod_time = parsedate_to_datetime(last_modified).timestamp()
-
+        
         if Path.exists(localPath):
             local_mod_time = localPath.stat().st_mtime
             if local_mod_time >= dump_mod_time:
                 self.tdenv.DEBUG0("'{}': Dump is not more recent than Local.", path)
                 return False
-
+        
         # The server doesn't know the gzip'd length, and we won't see the gzip'd data,
         # so we want the actual text-only length. Capture it here so we can tell the
         # transfer mechanism how big the file is going to be.
         length = response.headers.get("content-length")
-
+        
         self.tdenv.NOTE("Downloading file '{}'.", path)
         transfers.download(self.tdenv, url, localPath, chunkSize=16384, length=length)
-
+        
         # Change the timestamps on the file so they match the website
         os.utime(localPath, (dump_mod_time, dump_mod_time))
-
+        
         return True
-
+    
     def purgeSystems(self):
         """
         Purges systems from the System table that do not have any stations claiming to be in them.
         Keeps table from becoming too large because of fleet carriers moving to unpopulated systems.
         """
         self.tdenv.NOTE("Purging Systems with no stations: Start time = {}", self.now())
-
+        
         Session = self.tdb.Session
         with Session.begin() as session:
             subq = select(SA.Station.system_id).where(SA.Station.system_id == SA.System.system_id)
             stmt = delete(SA.System).where(~exists(subq))
             session.execute(stmt)
-
+        
         self.tdenv.NOTE("Finished purging Systems. End time = {}", self.now())
-
+    
     def importListings(self, listings_file):
         """
         Updates the market data (StationItem) using `listings_file`.
-
+        
         Rules:
           - If a row doesn't exist in DB → insert (copy CSV exactly).
           - If it exists → update only when CSV.modified > DB.modified.
@@ -207,40 +205,40 @@ class ImportPlugin(plugins.ImportPluginBase):
             end_bulk_mode,
             get_upsert_fn,
         )
-
+        
         listings_path = Path(self.dataPath, listings_file).absolute()
-        from_live = int(listings_path != Path(self.dataPath, self.listingsPath).absolute())
-
+        from_live = listings_path != Path(self.dataPath, self.listingsPath).absolute()
+        
         self.tdenv.NOTE("Checking listings")
         total = _count_listing_entries(self.tdenv, listings_path)
         if not total:
             self.tdenv.NOTE("No listings")
             return
-
+        
         self.tdenv.NOTE(
             "Processing market data from {}: Start time = {}. Live = {}",
-            listings_file, self.now(), bool(from_live)
+            listings_file, self.now(), from_live
         )
-
+        
         Session = self.tdb.Session
-
+        
         # Prefetch item/station IDs for early filtering
         with Session.begin() as session:
             item_lookup = _make_item_id_lookup(self.tdenv, session)
             station_lookup = _make_station_id_lookup(self.tdenv, session)
-
+        
         is_debug = self.tdenv.debug > 0
         self.tdenv.DEBUG0("Processing entries...")
-
+        
         with pbar.Progress(total, 40, prefix="Processing", style=pbar.LongRunningCountBar) as prog, \
-             listings_path.open("r", encoding="utf-8", errors="ignore") as fh, \
-             Session() as session:
-
+               listings_path.open("r", encoding="utf-8", errors="ignore") as fh, \
+               Session() as session:
+            
             token = begin_bulk_mode(session, profile="eddblink", phase="incremental")
             try:
                 commit_batch = get_import_batch_size(session, profile="eddblink")
                 execute_batch = commit_batch or 10000  # cap statement size even if single final commit
-
+                
                 # Upsert: keys + guarded fields (including from_live), guarded by 'modified'
                 table = SA.StationItem.__table__
                 key_cols = ("station_id", "item_id")
@@ -257,29 +255,29 @@ class ImportPlugin(plugins.ImportPluginBase):
                     modified_col="modified",
                     always_update=(),   # IMPORTANT: no unconditional updates
                 )
-
+                
                 batch_rows = []
                 since_commit = 0
-
+                
                 for listing in csv.DictReader(fh):
                     prog.increment(1)
                     try:
                         station_id = int(listing["station_id"])
                         if station_id not in station_lookup:
                             continue
-
+                        
                         item_id = int(listing["commodity_id"])
                         if item_id not in item_lookup:
                             continue  # skip rare items (not in Item table)
-
+                        
                         listing_time = int(listing["collected_at"])
                         dt_listing_time = datetime.datetime.utcfromtimestamp(listing_time)
-
+                        
                         row = {
-                            "station_id": station_id,
-                            "item_id": item_id,
-                            "modified": dt_listing_time,   # guard column
-                            "from_live": from_live,        # copied exactly when updating/inserting
+                            "station_id":   station_id,
+                            "item_id":      item_id,
+                            "modified":     dt_listing_time,   # guard column
+                            "from_live":    int(from_live),        # copied exactly when updating/inserting
                             "demand_price": int(listing["sell_price"]),
                             "demand_units": int(listing["demand"]),
                             "demand_level": int(listing.get("demand_bracket") or "-1"),
@@ -289,44 +287,43 @@ class ImportPlugin(plugins.ImportPluginBase):
                         }
                         batch_rows.append(row)
                         since_commit += 1
-
+                        
                         if len(batch_rows) >= execute_batch:
                             upsert(batch_rows)
                             batch_rows.clear()
-
+                        
                         if commit_batch and since_commit >= commit_batch:
                             session.commit()
                             since_commit = 0
-
+                    
                     except Exception as e:  # pylint: disable=broad-exception-caught
                         self.tdenv.WARN("Bad listing row (skipped): {}  error: {}", listing, e)
                         continue
-
+                
                 if batch_rows:
                     upsert(batch_rows)
                     batch_rows.clear()
-
+                
                 session.commit()
-
+            
             finally:
                 end_bulk_mode(session, token)
-
-        with pbar.Progress(1, 40, prefix="Saving"):
-            pass
-
+        
+        # with pbar.Progress(1, 40, prefix="Saving"):
+        #     pass
+        
         if self.getOption("optimize"):
             with pbar.Progress(1, 40, prefix="Optimizing"):
                 if self.tdb.engine.dialect.name == "sqlite":
                     with Session.begin() as session:
                         session.execute(text("VACUUM"))
-
+        
         self.tdenv.NOTE("Finished processing market data. End time = {}", self.now())
-
-
+    
     def run(self):
         """
         EDDN/EDDB link importer.
-
+        
         Refactored DB flow:
           - No dialect-specific logic in the plugin.
           - Preflight uses TradeDB.reloadCache() (which centralizes sanity via lifecycle.ensure_fresh_db).
@@ -340,10 +337,10 @@ class ImportPlugin(plugins.ImportPluginBase):
         from tradedangerous import cache
         # bulk-mode helpers for the incremental static import session
         from tradedangerous.db.utils import begin_bulk_mode, end_bulk_mode
-
+        
         self.tdenv.ignoreUnknown = True
         self.tdb.dataPath.mkdir(parents=True, exist_ok=True)
-
+        
         # Enable 'listings' by default unless other explicit options are present
         default = True
         for option in self.options:
@@ -351,10 +348,12 @@ class ImportPlugin(plugins.ImportPluginBase):
                 default = False
         if default:
             self.options["listings"] = True
-
-        # -----------------------------
-        # Optional CLEAN: prepare inputs
-        # -----------------------------
+        
+        # Check if database already exists and enable `clean` if not.
+        from tradedangerous.db.lifecycle import is_empty
+        if is_empty(self.tdb.engine):
+            self.options["clean"] = True
+        
         if self.getOption("clean"):
             # Remove CSVs so downloads become the new source of truth
             for name in [
@@ -369,49 +368,39 @@ class ImportPlugin(plugins.ImportPluginBase):
                     os.remove(str(f))
                 except FileNotFoundError:
                     pass
-
-            # Remove .prices (will be regenerated later)
+            
+            # Remove .prices (DEPRECATED)
             try:
                 os.remove(str(self.tdb.dataPath / "TradeDangerous.prices"))
             except FileNotFoundError:
                 pass
-
-            # Stash RareItem.csv so a full rebuild doesn't hit FK issues
-            self._ri_path = self.tdb.dataPath / "RareItem.csv"
-            self._rib_path = self._ri_path.with_suffix(".tmp")
-            if self._ri_path.exists():
-                if self._rib_path.exists():
-                    self._rib_path.unlink()
-                self._ri_path.rename(self._rib_path)
-
-            # Full update after downloads
+                        
             self.options["all"] = True
             self.options["force"] = True
-
-        # --------------------------------
-        # Option cascade (unchanged logic)
-        # --------------------------------
+        
+        
+        # Select which options will be updated
         if self.getOption("listings"):
             self.options["item"] = True
             self.options["station"] = True
-
+        
         if self.getOption("shipvend"):
             self.options["ship"] = True
             self.options["station"] = True
-
+        
         if self.getOption("upvend"):
             self.options["upgrade"] = True
             self.options["station"] = True
-
+        
         if self.getOption("item"):
             self.options["station"] = True
-
+        
         if self.getOption("rare"):
             self.options["station"] = True
-
+        
         if self.getOption("station"):
             self.options["system"] = True
-
+        
         if self.getOption("all"):
             self.options["item"] = True
             self.options["rare"] = True
@@ -422,205 +411,71 @@ class ImportPlugin(plugins.ImportPluginBase):
             self.options["upgrade"] = True
             self.options["upvend"] = True
             self.options["listings"] = True
-
+        
         if self.getOption("solo"):
             self.options["listings"] = False
             self.options["skipvend"] = True
-
+        
         if self.getOption("skipvend"):
             self.options["shipvend"] = False
             self.options["upvend"] = False
-
-        # ---------------------------------------------
-        # Downloads — track which static CSVs changed
-        # ---------------------------------------------
-        changed = {
-            "System": False,
-            "Station": False,
-            "Category": False,
-            "Item": False,
-            "RareItem": False,
-            "Ship": False,
-            "ShipVendor": False,
-            "Upgrade": False,
-            "UpgradeVendor": False,
-            "FDevShipyard": False,
-            "FDevOutfitting": False,
-        }
-
-        # EDCD mirrors
+        
+        # Download required files and update tables.
+        buildCache = False
         if self.getOption("upgrade"):
             if self.downloadFile(self.upgradesPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlOutfitting, self.FDevOutfittingPath)
-                changed["Upgrade"] = True
-                changed["FDevOutfitting"] = True
-
+                buildCache = True
+        
         if self.getOption("ship"):
             if self.downloadFile(self.shipPath) or self.getOption("force"):
                 transfers.download(self.tdenv, self.urlShipyard, self.FDevShipyardPath)
-                changed["Ship"] = True
-                changed["FDevShipyard"] = True
-
-        # Core static tables
+                buildCache = True
+        
         if self.getOption("rare"):
             if self.downloadFile(self.rareItemPath) or self.getOption("force"):
-                changed["RareItem"] = True
-
+                buildCache = True
+        
         if self.getOption("shipvend"):
             if self.downloadFile(self.shipVendorPath) or self.getOption("force"):
-                changed["ShipVendor"] = True
-
+                buildCache = True
+        
         if self.getOption("upvend"):
             if self.downloadFile(self.upgradeVendorPath) or self.getOption("force"):
-                changed["UpgradeVendor"] = True
-
+                buildCache = True
+        
         if self.getOption("system"):
             if self.downloadFile(self.sysPath) or self.getOption("force"):
-                changed["System"] = True
-
+                buildCache = True
+        
         if self.getOption("station"):
             if self.downloadFile(self.stationsPath) or self.getOption("force"):
-                changed["Station"] = True
-
+                buildCache = True
+        
         if self.getOption("item"):
             if self.downloadFile(self.commoditiesPath) or self.getOption("force"):
                 self.downloadFile(self.categoriesPath)
-                changed["Item"] = True
-                changed["Category"] = True
-
-        # -------------------------------------------------------------
-        # Preflight sanity (user-visible): make the pause explicit
-        # -------------------------------------------------------------
-        ri_path = getattr(self, "_ri_path", self.tdb.dataPath / "RareItem.csv")
-        rib_path = getattr(self, "_rib_path", ri_path.with_suffix(".tmp"))
-        rareitem_stashed = False
-        self.tdenv.NOTE("Preflight: verifying database (this can take a while on first run)...")
-        t0 = time.monotonic()
-        try:
-            if ri_path.exists():
-                if not rib_path.exists() and not self.getOption("clean"):
-                    ri_path.rename(rib_path)
-                    rareitem_stashed = True
-
-            # This may no-op or may call buildCache() internally
-            self.tdb.reloadCache()
-        finally:
-            if rib_path.exists() and (self.getOption("clean") or rareitem_stashed):
-                if ri_path.exists():
-                    ri_path.unlink()
-                rib_path.rename(ri_path)
-            t1 = time.monotonic()
-            self.tdenv.NOTE("Preflight complete in {:.1f}s.", (t1 - t0))
-
-        # -----------------------------------------------------
-        # Rebuild or Incremental Import?
-        # -----------------------------------------------------
-        if self.getOption("clean"):
-            self.tdenv.NOTE("Performing full rebuild...")
+                buildCache = True
+        
+        # Remake the .db files with the updated info.
+        if buildCache:
             self.tdb.close()
-            cache.buildCache(self.tdb, self.tdenv)
+            self.tdb.reloadCache()
             self.tdb.close()
-            self.tdenv.NOTE("Full rebuild complete.")
-        else:
-            # Incremental import of only changed tables (no schema drop)
-            IMPORT_ORDER = [
-                "System",
-                "Station",
-                "Category",
-                "Item",
-                "RareItem",
-                "Ship",
-                "ShipVendor",
-                "Upgrade",
-                "UpgradeVendor",
-                "FDevShipyard",
-                "FDevOutfitting",
-            ]
-
-            any_changed = any(changed.values())
-            if any_changed:
-                self.tdenv.NOTE("Incremental import starting ({} tables changed).", sum(1 for v in changed.values() if v))
-                with self.tdb.Session() as session:
-                    token = begin_bulk_mode(session, profile="eddblink", phase="incremental")
-                    try:
-                        for table_name in IMPORT_ORDER:
-                            if not changed.get(table_name):
-                                continue
-                            import_path = (self.tdb.dataPath / f"{table_name}.csv").resolve()
-                            try:
-                                # Determine a cheap per-table total (header-aware) for display only.
-                                try:
-                                    total = max(file_line_count(import_path) - 1, 0)
-                                except Exception:
-                                    total = 0
-
-                                prefix = f"Processing {table_name}"
-                                # Mirror listings-style progress: single-line if TTY, periodic otherwise.
-                                with pbar.Progress(total or 1, 40, prefix=prefix, style=pbar.LongRunningCountBar) as prog:
-
-                                    def _cb(stats=None, **kwargs):
-                                        """
-                                        Liberal progress callback used by cache.processImportFile.
-                                        Accepts either:
-                                          - int                    → increment by that many rows
-                                          - dict with keys inc/rows/count → increment by that value
-                                          - anything else          → default increment of 1
-                                        """
-                                        inc = 1
-                                        if isinstance(stats, int):
-                                            inc = max(int(stats), 1)
-                                        elif isinstance(stats, dict):
-                                            for k in ("inc", "rows", "count"):
-                                                if k in stats:
-                                                    try:
-                                                        inc = max(int(stats[k]), 1)
-                                                        break
-                                                    except Exception:
-                                                        pass
-                                        prog.increment(inc)
-
-                                    cache.processImportFile(
-                                        self.tdenv,
-                                        session,
-                                        import_path,
-                                        table_name,
-                                        line_callback=_cb,
-                                        call_args={"table": table_name, "total": total},
-                                    )
-
-                                session.commit()
-                                self.tdenv.DEBUG0("Incremental import OK: {} ({})", table_name, import_path)
-
-                            except FileNotFoundError:
-                                self.tdenv.NOTE("{} missing; skipped incremental import ({})", table_name, import_path)
-                            except StopIteration:
-                                self.tdenv.NOTE("{} exists but is empty; skipped incremental import ({})", table_name, import_path)
-                            except Exception as e:
-                                self.tdenv.WARN("Incremental import failed for {}: {} ({})", table_name, e, import_path)
-                                session.rollback()
-                                self.tdenv.NOTE("Escalating to full rebuild due to import failure.")
-                                self.tdb.close()
-                                cache.buildCache(self.tdb, self.tdenv)
-                                self.tdb.close()
-                                break
-                    finally:
-                        end_bulk_mode(session, token)
-                self.tdenv.NOTE("Incremental import finished.")
-
-
+        
         if self.getOption("purge"):
             self.purgeSystems()
-
+        
         # Listings import (prices)
         if self.getOption("listings"):
             if self.downloadFile(self.listingsPath) or self.getOption("force"):
                 self.importListings(self.listingsPath)
             if self.downloadFile(self.liveListingsPath) or self.getOption("force"):
                 self.importListings(self.liveListingsPath)
-
-        if self.getOption("listings"):
-            self.tdenv.NOTE("Regenerating .prices file.")
-            cache.regeneratePricesFile(self.tdb, self.tdenv)
-
+        
+        # if self.getOption("listings"):
+        #     self.tdenv.NOTE("Regenerating .prices file.")
+        #     cache.regeneratePricesFile(self.tdb, self.tdenv)
+        
         self.tdenv.NOTE("Import completed.")
         return False