tradedangerous 11.1.2__py3-none-any.whl → 11.1.3__py3-none-any.whl

tradedangerous/plugins/eddblink_plug.py

@@ -3,23 +3,22 @@ Import plugin that uses data files from
 https://elite.tromador.com/ to update the Database.
 """
 from __future__ import annotations
+
+from pathlib import Path
+from .. import plugins, cache, transfers
+from ..misc import progress as pbar
+from ..plugins import PluginException
+
 import certifi
 import csv
 import datetime
-import json
 import os
+import requests
 import sqlite3
 import ssl
-import time
 import typing
 import locale
-
-from urllib import request
-from pathlib import Path
-
-from .. import plugins, cache, transfers
-from ..misc import progress as pbar
-from ..plugins import PluginException
+import platform
 
 
 if typing.TYPE_CHECKING:
@@ -28,7 +27,6 @@ if typing.TYPE_CHECKING:
 
 # Find the first English UTF-8 locale for use in parsing timestamps.
 LOCALE = None
-import platform
 if platform.system() == 'Windows':
     for lang in locale.windows_locale.values():
         if "en" in lang:
@@ -52,14 +50,6 @@ BASE_URL = os.environ.get('TD_SERVER') or "https://elite.tromador.com/files/"
 CONTEXT=ssl.create_default_context(cafile=certifi.where())
 
 
-def _request_url(url, headers=None):
-    data = None
-    if headers:
-        data = bytes(json.dumps(headers), encoding="utf-8")
-    
-    return request.urlopen(request.Request(url, data=data), context=CONTEXT, timeout=90)
-
-
 class DecodingError(PluginException):
     pass
 
@@ -180,9 +170,6 @@ class ImportPlugin(plugins.ImportPluginBase):
         """
         Fetch the latest dumpfile from the website if newer than local copy.
         """
-        def openURL(url):
-            return _request_url(url, headers = {'User-Agent': 'Trade-Dangerous'})
-        
         if path not in (self.liveListingsPath, self.listingsPath):
             localPath = Path(self.tdb.dataPath, path)
         else:
@@ -191,26 +178,38 @@ class ImportPlugin(plugins.ImportPluginBase):
         url  = BASE_URL + str(path)
         
         self.tdenv.NOTE("Checking for update to '{}'.", path)
+        # Use an HTTP Request header to obtain the Last-Modified and Content-Length headers.
+        # Also, tell the server to give us the un-compressed length of the file by saying
+        # that >this< request only wants text.
+        headers = {"User-Agent": "Trade-Dangerous", "Accept-Encoding": "text"}
         try:
-            response = openURL(url)
+            response = requests.head(url, headers=headers, timeout=70)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", BASE_URL + str(path), str(e))
+            self.tdenv.WARN("Problem with download:\n    URL: {}\n    Error: {}", url, str(e))
             return False
         
-        url_time = response.getheader("Last-Modified")
         locale.setlocale(locale.LC_ALL, LOCALE)
-        dumpModded = datetime.datetime.strptime(url_time, "%a, %d %b %Y %H:%M:%S %Z").timestamp()
+        last_modified = response.headers.get("last-modified")
+        dump_mod_time = datetime.datetime.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z").timestamp()
         locale.setlocale(locale.LC_ALL, '')
         
         if Path.exists(localPath):
-            localModded = localPath.stat().st_mtime
-            if localModded >= dumpModded:
+            local_mod_time = localPath.stat().st_mtime
+            if local_mod_time >= dump_mod_time:
                 self.tdenv.DEBUG0("'{}': Dump is not more recent than Local.", path)
                 return False
         
+        # The server doesn't know the gzip'd length, and we won't see the gzip'd data,
+        # so we want the actual text-only length. Capture it here so we can tell the
+        # transfer mechanism how big the file is going to be.
+        length = response.headers.get("content-length")
+        
         self.tdenv.NOTE("Downloading file '{}'.", path)
-        transfers.download(self.tdenv, url, localPath)
-        os.utime(localPath, (dumpModded, dumpModded))
+        transfers.download(self.tdenv, url, localPath, chunkSize=16384, length=length)
+        
+        # Change the timestamps on the file so they match the website
+        os.utime(localPath, (dump_mod_time, dump_mod_time))
+        
         return True
     
     def purgeSystems(self):
@@ -355,6 +354,8 @@ class ImportPlugin(plugins.ImportPluginBase):
     def run(self):
         self.tdenv.DEBUG2(f'Using "{LOCALE}" locale for parsing modified timestamps. Please include this information in any error reports.')
         
+        self.tdenv.ignoreUnknown = True
+        
         # Create the /eddb folder for downloading the source files if it doesn't exist.
         try:
             Path(str(self.dataPath)).mkdir()
@@ -511,8 +512,6 @@ class ImportPlugin(plugins.ImportPluginBase):
             self.tdb.close()
             self.tdb.reloadCache()
         
-        self.tdenv.ignoreUnknown = True
-        
         if self.getOption("purge"):
             self.purgeSystems()
         

tradedangerous/tradedb.py

@@ -1150,8 +1150,9 @@ class TradeDB:
         self.tradingStationCount = 0
         # Fleet Carriers are station type 24.
         # Odyssey settlements are station type 25.
+        # Assume type 0 (Unknown) are also Fleet Carriers.
         # Storing as a list allows easy expansion if needed.
-        types = {'fleet-carrier':[24,],'odyssey':[25,],}
+        types = {'fleet-carrier':[24, 0,],'odyssey':[25,],}
         with closing(self.query(stmt)) as cur:
             for (
                 ID, systemID, name,

tradedangerous/transfers.py

@@ -1,6 +1,4 @@
-import csv
-import json
-import time
+from __future__ import annotations
 
 from collections import deque
 from pathlib import Path
@@ -8,9 +6,20 @@ from .tradeexcept import TradeException
 from .misc import progress as pbar
 from . import fs
 
+import csv
+import json
+import time
+import typing
+
 import requests
 
 
+if typing.TYPE_CHECKING:
+    import os  # for PathLike
+    from .tradeenv import TradeEnv
+    from typing import Optional, Union
+
+
 ######################################################################
 # Helpers
 
@@ -31,12 +40,16 @@ def makeUnit(value):
     return None
 
 def download(
-            tdenv, url, localFile,
-            headers=None,
-            backup=False,
-            shebang=None,
-            chunkSize=4096,
-            timeout=90,
+            tdenv:      TradeEnv,
+            url:        str,
+            localFile:  os.PathLike,
+            headers:    Optional[dict] = None,
+            backup:     bool = False,
+            shebang:    Optional[str] = None,
+            chunkSize:  int = 4096,
+            timeout:    int = 90,
+            *,
+            length:     Optional[Union[int, str]] = None,
         ):
     """
     Fetch data from a URL and save the output
@@ -57,21 +70,24 @@ def download(
     shebang:
         function to call on the first line
     """
-    
     tdenv.NOTE("Requesting {}".format(url))
+
+    if isinstance(length, str):
+        length = int(length)
+
     req = requests.get(url, headers=headers or None, stream=True, timeout=timeout)
     req.raise_for_status()
     
     encoding = req.headers.get('content-encoding', 'uncompress')
-    length = req.headers.get('content-length', None)
+    content_length = req.headers.get('content-length', length)
     transfer = req.headers.get('transfer-encoding', None)
-    if transfer != 'chunked':
+    if not length and transfer != 'chunked':
         # chunked transfer-encoding doesn't need a content-length
-        if length is None:
+        if content_length is None:
             print(req.headers)
             raise Exception("Remote server replied with invalid content-length.")
-        length = int(length)
-        if length <= 0:
+        content_length = int(content_length)
+        if content_length <= 0:
             raise TradeException(
                 "Remote server gave an empty response. Please try again later."
             )
@@ -81,8 +97,8 @@ def download(
     # the uncompressed data, which should be larger, which will cause our
     # download indicators to sit at 100% for a really long time if the file is
     # heavily compressed and large (e.g spansh 1.5gb compressed vs 9GB uncompressed)
-    if encoding == "gzip" and length:
-        length *= 4
+    if length is None and encoding == "gzip" and content_length:
+        length = content_length * 3
     
     if tdenv.detail > 1:
         if length:

tradedangerous/version.py

@@ -12,5 +12,5 @@
 """just keeper of current version"""
 
 # TODO: remember to update tests when version changes
-__version__ = '11.1.2'
+__version__ = '11.1.3'
 

{tradedangerous-11.1.2.dist-info → tradedangerous-11.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tradedangerous
-Version: 11.1.2
+Version: 11.1.3
 Summary: Trade-Dangerous is a set of powerful trading tools for Elite Dangerous, organized around one of the most powerful trade run optimizers available.
 Home-page: https://github.com/eyeonus/Trade-Dangerous
 Author: eyeonus

{tradedangerous-11.1.2.dist-info → tradedangerous-11.1.3.dist-info}/RECORD

@@ -16,12 +16,12 @@ tradedangerous/prices.py,sha256=JqiDVrtvvPd5pqE3HdwOHOuFgdAbOR-pt0GLD3ZIXM8,7425
 tradedangerous/submit-distances.py,sha256=zxHtRpX10soJKLuRcun9uelfXGq2S9jvIWxD--P9zus,11709
 tradedangerous/tools.py,sha256=pp-4WtA12SVaaQHFJFOMTF7EDFRCU2mQeOhC4xoXmEk,1331
 tradedangerous/tradecalc.py,sha256=A7peEMiaCjlwFvReSq3E7_Ar0shUoFedQi83ZmOc7uY,42075
-tradedangerous/tradedb.py,sha256=3nGB55dYs4igP3U3J4Ye1-M6Kt9A4xPAMmnX7JEDW7w,72220
+tradedangerous/tradedb.py,sha256=mitKkS4MczivDK_K7A-IC94hkObUmGWFhwIrh_ts9qw,72282
 tradedangerous/tradeenv.py,sha256=SDzRC6ERYZzzb_I6uexmFpFJJrnbzXa-1ogYt_GH26w,10576
 tradedangerous/tradeexcept.py,sha256=aZ-Y31MbkjF7lmAzBAbaMsPPE7FEEfuf4gaX2GvriDk,368
-tradedangerous/transfers.py,sha256=88gIvXpjd8T6NLxBrBRzmH2IfUmDDtiMcDTrc7qF3OI,7830
+tradedangerous/transfers.py,sha256=KwskYBB5_TYDXx5fkN1ABYsPuPnICKdkEx0WUfC7BnA,8378
 tradedangerous/utils.py,sha256=PUPvAEqUyxYGqqQa0b_yfLAvq8YVUxK6HfdS-CxM-Lo,5186
-tradedangerous/version.py,sha256=hLMV6gA7ZJhJzy7LVW80y_VlNi2W67U6jnuBSCnnxI8,646
+tradedangerous/version.py,sha256=ghxPDFkK4akB_kG7EbpxFhrsQPs8VjNqdIUWHOPecuc,646
 tradedangerous/commands/TEMPLATE.py,sha256=MOE69xsZPHPIMBQ-LXicfsOlCZdy-2gPX_nlnwYYil8,2026
 tradedangerous/commands/__init__.py,sha256=3gz2cnXNZNkV1gtZh0dOnCRxBkQHbeIyysRe3bM2WEE,9516
 tradedangerous/commands/buildcache_cmd.py,sha256=jhNSqHX_xX43SiSUMFiKtWpB9v4oeZ0sqfNq6DCrjUs,2181
@@ -62,7 +62,7 @@ tradedangerous/misc/progress.py,sha256=NKvKP1OSCTpItc1CNxDuEH2A1oGJ6aWSyCdPSAjsG
 tradedangerous/plugins/__init__.py,sha256=TL-OIptlqNENKhoFqkFeBJn_vSw8L0pVaDJgjhaTj7A,7860
 tradedangerous/plugins/edapi_plug.py,sha256=5nqBYmjUceAt-KTfiBn7IEl443R1SsGLDmfVXgbcyms,42262
 tradedangerous/plugins/edcd_plug.py,sha256=JuDtuEM_mN9Sz2H09-qYizM-9N3cuNjgvQy7Y-wHwKw,14412
-tradedangerous/plugins/eddblink_plug.py,sha256=0UZWiV7HgMBO8R0hDzDdNYKX6ZvF9_SjGq0RDcbBNJU,22484
+tradedangerous/plugins/eddblink_plug.py,sha256=xK5snHevw8BIFfpYNtrJYRQekDD0DYkp62kcxSrkZ-8,22883
 tradedangerous/plugins/edmc_batch_plug.py,sha256=rrP_lFFxWsba8DPEo0WF2EdCiMoRC7tCT8z62MIvtIo,4173
 tradedangerous/plugins/journal_plug.py,sha256=5HMyoxQ7z42qj7NiL8rDxSyTN9gKikoQjyWzJLD-SYQ,23746
 tradedangerous/plugins/netlog_plug.py,sha256=yUl47l9xt3kGj9oSiY_FZaDGdnQj63oa9MBtSeIy1Zo,13469
@@ -71,9 +71,9 @@ tradedangerous/templates/Added.csv,sha256=8o54civQCcS9y7_DBo0GX196XWRbbREQqKDYTK
 tradedangerous/templates/Category.csv,sha256=8xwUDcBZE25T6x6dZGlRUMTCqeDLt3a9LXU5h6hRHV8,250
 tradedangerous/templates/RareItem.csv,sha256=F1RhRnTD82PiwrVUO-ai2ErGH2PTqNnQaDw5mcgljXs,10483
 tradedangerous/templates/TradeDangerous.sql,sha256=VlQK7QGtEi2brGtWaIZDvKmbJ_vLocD4CJ8h_6kKptU,7808
-tradedangerous-11.1.2.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-tradedangerous-11.1.2.dist-info/METADATA,sha256=FYveEqHoQHlAuYrlNC3Cr32090JUEgq4rGEvgE52j30,4435
-tradedangerous-11.1.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tradedangerous-11.1.2.dist-info/entry_points.txt,sha256=lrA7U9JHOcNuam2WEK4Hmc3vQ3mrJfsbJCE74qd9au8,62
-tradedangerous-11.1.2.dist-info/top_level.txt,sha256=JEoOVAhg5GfXZ4kHpNontu0RVzek_7P9_jp93f3Pqn8,16
-tradedangerous-11.1.2.dist-info/RECORD,,
+tradedangerous-11.1.3.dist-info/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+tradedangerous-11.1.3.dist-info/METADATA,sha256=_YJd9yECrUzNkslTW4-PSLUoJPuS_EJIV3Dkxdymyq0,4435
+tradedangerous-11.1.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tradedangerous-11.1.3.dist-info/entry_points.txt,sha256=lrA7U9JHOcNuam2WEK4Hmc3vQ3mrJfsbJCE74qd9au8,62
+tradedangerous-11.1.3.dist-info/top_level.txt,sha256=JEoOVAhg5GfXZ4kHpNontu0RVzek_7P9_jp93f3Pqn8,16
+tradedangerous-11.1.3.dist-info/RECORD,,