tradedangerous 12.7.6__py3-none-any.whl

tradedangerous/tradeorm.py

@@ -0,0 +1,183 @@
+"""
+tradeorm provides the TradeORM class which uses the application database
+rather than trying to be its own database in its own right like TradeDB.
+
+Suggested use:
+    
+    # TradeEnv is optional, it's for controlling environment settings
+    # builder-pattern style.
+    from tradedangerous import TradeEnv, TradeORM
+    
+    tde = TradeEnv()  # debug settings, color, etc...
+    tdo = TradeORM(tde)  # if not supplied, it will make its own
+"""
+from __future__ import annotations
+from pathlib import Path
+import os
+import typing
+
+from . import TradeEnv
+from .tradeexcept import AmbiguityError, TradeException, MissingDB, SystemNotStationError
+from .db import (
+    orm_models as orm,          # type: ignore  # so we can access models easily
+    make_engine_from_config,    # type: ignore
+    get_session_factory,        # type: ignore
+)
+
+if typing.TYPE_CHECKING:
+    from .db.engine import sessionmaker, Engine, Session  # type: ignore
+
+
+class TradeORM:
+    DEFAULT_PATH = "data"
+    DEFAULT_DB = "TradeDangerous.db"
+    DB_CONFIG_VAR = "TD_DB_CONFIG"
+    DB_CONFIG_FILE = "db_config.ini"
+    
+    data_dir: Path
+    db_path:  Path
+    
+    engine: Engine
+    session_maker: sessionmaker[Session]
+    session: Session
+    
+    def __init__(self, *, tdenv: TradeEnv | None = None, debug: int | None = None):
+        tdenv = tdenv or TradeEnv(debug=debug or 0)
+        self.tdenv = tdenv
+        
+        # Determine where the database should be
+        data_dir = tdenv.dataDir or TradeORM.DEFAULT_PATH
+        self.data_dir = Path(data_dir)
+        tdenv.DEBUG0("data_dir = {}", self.data_dir)
+        
+        # Determine the path to the file itself
+        db_path = tdenv.dbFilename or (self.data_dir / TradeORM.DEFAULT_DB)
+        self.db_path  = Path(db_path)
+        tdenv.DEBUG0("db_path = {}", self.db_path)
+        
+        # We need it to exist.
+        if not self.db_path.exists():
+            raise MissingDB(self.db_path)
+        
+        default_config = self.data_dir / TradeORM.DB_CONFIG_FILE
+        db_config = os.environ.get(TradeORM.DB_CONFIG_VAR, default_config)
+        tdenv.DEBUG0("db_config = {}", db_config)
+        
+        # Make the database available.
+        self.engine = make_engine_from_config(db_config)
+        
+        # The user will expect objects (instances of models) that we return
+        # to have the same lifetime as the TradeORM() instance, so we want
+        # a main session for things to use and return from.
+        #
+        # However: we also want them to be able to create transactions, etc
+        # so we also make the session-factory available.
+        self.session = get_session_factory(self.engine)()
+    
+    def commit(self):
+        """ Commit the current transaction state. """
+        return self.session.commit()
+    
+    def lookup_station(self, name: str) -> orm.Station | None:
+        """ Use the database to lookup a station, which accepts a name that
+            is either a unique station name (or partial of one), or in the
+            'system name/station name' component. If the station does not
+            match a unique station, raises an AmbiguityError
+        """
+        if "%" in name:
+            raise TradeException("wildcards ('%') are not supported in station names")
+        if "/" not in name:
+            if (station := self._station_lookup(name, exact=True, partial=False)):
+                return station
+            if self._system_lookup(name, exact=True, partial=False):
+                raise SystemNotStationError(f'"{name}" is a system name, use "/{name}" if you meant it as a station')
+            name = "/" + name
+        station: orm.Station | None = self.lookup_place(name)
+        return station
+    
+    def lookup_system(self, name: str) -> orm.System | None:
+        """ Use the database to lookup a system, which accepts a name that
+            is either a unique system name (or partial of one), or in the
+            'system name/station name' component. If the system does not
+            match a unique system, raises an AmbiguityError
+        """
+        if "%" in name:
+            raise TradeException("wildcards ('%') are not supported in system names")
+        system_name, _, _ = name.partition("/")
+        if not system_name:
+            raise TradeException(f"system name required for system lookup, got {name}")
+        result: orm.Station | orm.System | None = self.lookup_place(system_name)
+        if isinstance(result, orm.Station):
+            return result.system
+        return result
+    
+    def lookup_place(self, name: str) -> orm.Station | orm.System | None:
+        """ Using a "[<system>]/[<station>]" style name, look up either a Station or a System."""
+        if "%" in name:
+            raise TradeException("wildcards ('%') are not supported in names")
+        sys_name, slashed, stn_name = name.partition("/")
+        if not slashed:
+            if stn_name:
+                station: orm.Station | None = self._station_lookup(stn_name, exact=True, partial=False)
+                if station:
+                    return station
+            if sys_name:
+                system: orm.System | None = self._system_lookup(sys_name, exact=True, partial=False)
+                if system:
+                    return system
+        
+        if sys_name:
+            system = self._system_lookup(sys_name)
+            if not system:
+                raise TradeException(f"unknown system: {sys_name}")
+            if not stn_name:
+                return system
+            
+            # Now we match the list of station names for this system.
+            stmt = self.session.query(orm.Station).filter(orm.Station.system_id == system.system_id).filter(orm.Station.name == stn_name)
+            results = stmt.all()
+            if len(results) == 1:
+                return results[0]
+            stmt = self.session.query(orm.Station).filter(orm.Station.system_id == system.system_id).filter(orm.Station.name.like(f"%{stn_name}%"))
+            results = stmt.all()
+            if len(results) > 1:
+                raise AmbiguityError("Station", stn_name, [s.name for s in results])
+            return results[0]
+        
+        station = self._station_lookup(stn_name, exact=False)
+        return station
+    
+    def _system_lookup(self, name: str, *, exact: bool = True, partial: bool = True) -> orm.System | None:
+        """ Look up a model by exact name match. """
+        assert exact or partial, "at least one of exact or partial must be True"
+        results: list[orm.System] | None = None
+        if exact:
+            results = self.session.query(orm.System).filter(orm.System.name == name).all()
+            if len(results) == 1:
+                partial = False
+        if partial:
+            like_pattern = f"%{name}%"
+            results = self.session.query(orm.System).filter(orm.System.name.like(like_pattern)).all()
+        
+        if not results:
+            return None
+        if len(results) > 1:
+            raise AmbiguityError("System", name, results, key=lambda s: s.dbname())
+        return results[0]
+    
+    def _station_lookup(self, name: str, *, exact: bool = True, partial: bool = True) -> orm.Station | None:
+        """ Look up a model by exact name match. """
+        assert exact or partial, "at least one of exact or partial must be True"
+        results: list[orm.Station] | None = None
+        if exact:
+            results = self.session.query(orm.Station).filter(orm.Station.name == name).all()
+            if len(results) == 1:
+                partial = False
+        if partial:
+            like_pattern = f"%{name}%"
+            results = self.session.query(orm.Station).filter(orm.Station.name.like(like_pattern)).all()
+        if not results:
+            return None
+        if len(results) > 1:
+            raise AmbiguityError("Station", name, results, key=lambda s: s.dbname())
+        return results[0]

tradedangerous/transfers.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse, unquote
+
+from .tradeexcept import TradeException
+from .misc import progress as pbar
+from . import fs
+
+import json
+import time
+import typing
+
+import requests
+
+
+if typing.TYPE_CHECKING:
+    from collections.abc import Callable
+    import os  # for PathLike
+    from .tradeenv import TradeEnv
+
+
+######################################################################
+# Helpers
+
+class HTTP404(TradeException):
+    pass
+
+
+def makeUnit(value: float) -> str:
+    num, unit = split_unit(value)
+    return f"{num}{unit}"
+
+
+def split_unit(value: float) -> tuple[str, str]:
+    """
+    Split a byte size into a (number_str, unit_str) tuple.
+    Used when you need to colour or format the numeric part separately.
+    
+    Example:
+        >>> split_unit(30200000)
+        ('28.8', 'MB')
+    """
+    units = ["B", "KB", "MB", "GB", "TB"]
+    n = float(value)
+    i = 0
+    while n >= 1024.0 and i < len(units) - 1:
+        n /= 1024.0
+        i += 1
+    num_str = f"{n:.1f}" if i > 0 else f"{int(n)}"
+    return num_str, units[i]
+
+
+def get_filename_from_url(url: str) -> str:
+    """ extracts just the filename from a url. """
+    return Path(unquote(urlparse(url).path)).name
+
+
+def download(
+            tdenv:      TradeEnv,
+            url:        str,
+            localFile:  os.PathLike,
+            headers:    dict | None = None,
+            backup:     bool = False,
+            shebang:    Callable | None = None,
+            chunkSize:  int = 4096,
+            timeout:    int = 30,
+            *,
+            length:     int | str | None = None,
+            session:    requests.Session | None = None,
+        ):
+    """
+    Fetch data from a URL and save the output
+    to a local file. Returns the response headers.
+    
+    :param tdenv:       TradeEnv we're working under
+    :param url:         URL we're fetching (http, https or ftp)
+    :param localFile:   Name of the local file to open.
+    :param headers:     dict() of additional HTTP headers to send
+    :param shebang:     function to call on the first line
+    """
+    tdenv.NOTE("Requesting {}", url)
+    
+    if isinstance(length, str):
+        length = int(length)
+    
+    # If the caller provided an existing session stream, use that the fetch the request.
+    req = (session or requests).get(url, headers=headers or None, stream=True, timeout=timeout)
+    req.raise_for_status()
+    
+    encoding = req.headers.get('content-encoding', 'uncompress')
+    content_length = req.headers.get('content-length', length)
+    transfer = req.headers.get('transfer-encoding', None)
+    if not length and transfer != 'chunked':
+        # chunked transfer-encoding doesn't need a content-length
+        if content_length is None:
+            print(req.headers)
+            raise TradeException("Remote server replied with invalid content-length.")
+        content_length = int(content_length)
+        if content_length <= 0:
+            raise TradeException(
+                "Remote server gave an empty response. Please try again later."
+            )
+    
+    # if the file is being compressed by the server, the headers tell us the
+    # length of the compressed data, but in our loop below we will be receiving
+    # the uncompressed data, which should be larger, which will cause our
+    # download indicators to sit at 100% for a really long time if the file is
+    # heavily compressed and large (e.g spansh 1.5gb compressed vs 9GB uncompressed)
+    if length is None and encoding == "gzip" and content_length:
+        length = content_length * 3
+    
+    if tdenv.detail > 1:
+        if length:
+            tdenv.NOTE("Downloading {} {}ed data", makeUnit(length), encoding)
+        else:
+            tdenv.NOTE("Downloading {} {}ed data", transfer, encoding)
+    tdenv.DEBUG0(str(req.headers).replace("{", "{{").replace("}", "}}"))
+    
+    actPath = Path(localFile)
+    fs.ensurefolder(tdenv.tmpDir)
+    tmpPath = Path(tdenv.tmpDir, f"{actPath.name}.dl")
+    
+    fetched = 0
+    started = time.time()
+    filename = get_filename_from_url(url)
+    with pbar.Progress(max_value=length, width=25, prefix=filename, style=pbar.CountingBar, show=not tdenv.quiet) as prog, tmpPath.open("wb") as fh:
+        for data in req.iter_content(chunk_size=chunkSize):
+            fh.write(data)
+            fetched += len(data)
+            if shebang:
+                bangLine = data.decode().partition("\n")[0]
+                tdenv.DEBUG0("Checking shebang of {}", bangLine)
+                shebang(bangLine)
+                shebang = None
+            if prog:
+                prog.increment(len(data))
+        tdenv.DEBUG0("End of data")
+    
+    if not tdenv.quiet:
+        elapsed = (time.time() - started) or 1
+        num1, unit1 = split_unit(fetched)
+        num2, unit2 = split_unit(fetched / elapsed)
+        tdenv.NOTE(
+            f"Downloaded [cyan]{num1}[/]{unit1} of {encoding}ed data "
+            f"[cyan]{num2}[/]{unit2}/s"
+        )
+    
+    
+    fs.ensurefolder(actPath.parent)
+    
+    # Swap the file into place
+    if backup:
+        bakPath = Path(localFile + ".bak")
+        if bakPath.exists():
+            bakPath.unlink()
+        if actPath.exists():
+            actPath.rename(localFile + ".bak")
+    if actPath.exists():
+        actPath.unlink()
+    tmpPath.rename(actPath)
+    
+    req.close()
+    return req.headers
+
+def get_json_data(url, *, timeout: int = 90):
+    """
+    Fetch JSON data from a URL and return the resulting dictionary.
+    
+    Displays a progress bar as it downloads.
+    """
+    
+    req = requests.get(url, stream=True, timeout=timeout)
+    
+    totalLength = req.headers.get('content-length')
+    if totalLength is None:
+        compression = req.headers.get('content-encoding')
+        compression = (compression + "'ed") if compression else "uncompressed"
+        print("Downloading {}: {}...".format(compression, url))
+        jsData = req.content
+    else:
+        totalLength = int(totalLength)
+        filename = get_filename_from_url(url)
+        progBar = pbar.Progress(totalLength, 25, prefix=filename)
+        
+        jsData = bytes()
+        for data in req.iter_content():
+            jsData += data
+            progBar.increment(len(data))
+        progBar.clear()
+    
+    return json.loads(jsData.decode())

tradedangerous/utils.py

@@ -0,0 +1,243 @@
+# --------------------------------------------------------------------
+# Copyright (C) Oliver 'kfsone' Smith 2014 <oliver@kfs.org>:
+# Copyright (C) Bernd 'Gazelle' Gollesch 2016, 2017
+# Copyright (C) Jonathan 'eyeonus' Jones 2018, 2019
+#
+# You are free to use, redistribute, or even print and eat a copy of
+# this software so long as you include this copyright notice.
+# I guarantee there is at least one bug neither of us knew about.
+# --------------------------------------------------------------------
+# TradeDangerous :: Modules :: Utils
+#
+"""This modules contains various utils to be used internally.
+"""
+
+import re
+
+__all__ = ['titleFixup', 'checkForOcrDerp']
+
+ocrDerp = re.compile(r"""(
+    ^.$ |
+    LAN[O0]ING |
+    [O0][O0]CK |
+    [O0]INEILL |
+    AQUIRE[O0] |
+    FNT[EF]RPRIS[EF] |
+    [EF]NTFRPRIS[EF] |
+    [EF]NT[EF]RPRISF |
+    [O0](UTT|ALT)[O0]N |
+    8RA[DO0]LEY |
+    BRA[O0]LEY |
+    LLOY[O0] |
+    [O0]RBDAL |
+    DRB[O0]DAL |
+    [D0]RBITAL |
+    REE[O0] |
+    \BDOCK$ |
+    \BTERMINAL\b |
+    \bKID?[O0] |
+    \b[O3]E\b |
+    \bANDRA[O3]E\b |
+    \bAN[O3]RADE\b |
+    \bAN[O3]RA[O3]E\b |
+    VVELL\b |
+    [O0]IRAC\b |
+    \bVV |
+    \b[O0]ER?\b |
+    \b[O0]RAKE |
+    HAR[O0](T\b|W[I1L]CK) |
+    ACQU[I1L]R[E3][O0] |
+    \b[O0]ARK |
+    \b[O0]DAM |
+    [O0]EPOT |
+    \bMERE[O0] |
+    \b[O0]ENN?IS |
+    \bBRAN[o0] |
+    W[O0]{3} |
+    GO(D[O0]|[O0]D|[O0][O])ARD |
+    GO[DO0]{2}AR[O0] |
+    ORB[RH]AL\b |
+    \bJOR[O0]A |
+    \bST[O0]ART |
+    \bQUIMPY |
+    \bVAR[O0]E |
+    EN[^T]?ERPRISE |
+    EN..ERPRISE |
+    [E38]NT[E38]F[I1']?PR[I1'][S5][E38] |
+    \bMUR[O0]O |
+    \bBAR[O0]E |
+    \bBALLAR[O0] |
+    \b[O0]REYER\b |
+    \bEDWAR[O0] |
+    \bE[O0]WAR[DO0] |
+    III |
+    STARON\b |
+    \bST.ION\b |
+    \bSTION\b |
+    \BHANG[EA]R$ |
+    ^\S+HUB$ |
+    \bLEBEOEV |
+    \B(
+        BASE |
+        ENTE[RP]P[RP]ISE |
+        TERMINA(L|II) |
+        P(L|II)ANT |
+        RELAY |
+        ORBITAL |
+        PLATFORM |
+        COLONY |
+        VISION |
+        REFINERY
+    )$ |
+    [O0]RB[I1]DAL |
+    [O0]R[DL][I1]TAL |
+    \bBRIOGER |
+    \bJUOSON |
+    LANOER |
+    G[O0][O0]([RW]|VV)[O0I]N |
+    \bSPE([O0][DO0]|[DO0][O0])ING\b |
+    \bARCHIMEOES\b |
+    \bH[O0D]L[O0]ING |
+    \bM[O0D]HMAN[O0] |
+    \b[O0]ANA\b |
+    \bALEKSAN[O0]R[O0D]V\b |
+    \bCH[0D]MSKY\b |
+    \b[O0]IESEL\b |
+    [O0]{3} |
+    SCHMI[O0]T |
+    \bSAUN[O0]ER |
+    [O0]IV[E3] |
+    VIRIDN$ |
+    \bHORI\.ONS |
+    C[O0D]+LNY$ |
+    \bR[O0]ZH[O0]E[S5]TVENSKY |
+    \bRDZH[DO0]ESTVENSKY |
+    '' |
+    ^[^A-Z0-9] |
+    \s{2,} |
+    ^OEN |
+    ^MCK(EF|FE)\b |
+    \bCHAN\s+DLER |
+    \b[O0]UMONT |
+    \bUN[0O]ER |
+    \bSDMM |
+    \bREA([O0]D|D[O0]) |
+    \bRD[DO0][DO0]EN |
+    \bR[O0]([O0]D|D[O0])EN |
+    (?<!BR)[O0]ECK$ |
+    SE?TTL(FMENT|EMFNT|FMFNT) |
+    SE?TTLMENT |
+    STTL[E38]?M[E38]?NT |
+    S\s?[E38]\s?T\s?T\s?L\s?[E38](\sM|M\s)[E38]\s?N\s?T$ |
+    S[E38]TT[38]?L[E38]MNT |
+    MARKFT |
+    HANGFR |
+    CL(EVF|FVE|FVF) |
+    \bCRY$ |
+    G[E3][D0]RG[E3]LUCA[S5] |
+    PLAT[E3]F[D0]RM |
+    OCONNOR |
+    ` |
+    -- |
+    \bREILLI\b |
+    \bRIN[FC]\b |
+    \bOL[E3]ARY |
+    \bSATION\b |
+    ,\w |
+    \bI?NGLY\b |
+    \bAU\sL[DO0]\b |
+    (^|\s)['.] |
+    ^- | -$ |
+    \bDREBBFL\b
+    \bLEVIE |
+    \bRN\b |
+    \bH\sUNZIKER |
+    \bL[O0D]FTH\sUS |
+    \bHORNUCH\b |
+    \bKLU\sDZE |
+    ^[DR]HN\b |
+    SU\sI?RVEY$ |
+    H[DO0]L[O0]ING |
+    H[D0]LDING |
+    M[DO0]HMAN[O0] |
+    \bABL\b |
+    \bBENNET\b |
+    \bHU8\b |
+    \sCITV$ |
+    \sPIT[VY]$ |
+    \bTFR |
+    IVII |
+    \BINAI$ |
+    SET[IT]''LEMEN |
+    I'L | R'I | (^|\s)'L | [^Ss]'(?=\s|$) |
+    ^I \s (?! [Ss][Oo][Ll][Aa]) |
+    \bA7\S |
+    \sH\sI?UB$ |
+    \bALEDNDRIA\b |
+    \sH\sU\sB$ |
+    \bC[O0]LCNY\b |
+    \bOOCTE\b |
+    \bBULGAFIIN\b |
+    \bWH\sIEEL |
+    \bBR8NNAN\b |
+    \b(ID)?ING$ |
+    GATEVVAY$ |
+    [HI\s]U\sI?B$ |
+    CLAI\sI?M$ |
+    \bUITY$ |
+    \bDING$ |
+    BTOP$ |
+    B'I'OP$ |
+    TRANQUNUTY$ |
+    C[O0]LUNY$ |
+    \bMAGN\sI?US\b |
+    \s-[A-Z] |
+    ^NAKAM(\sIU|U\sIR) |
+    ^THOM\sI?PSON |
+    ^STEPH\sI?ENSON |
+    \bCQRK\b |
+    ^AN\sI?DREW |
+    ^WATSO\sIN |
+    ^QSSW |
+    ^RTZEN |
+    \bI?NAL$ |
+    \b''I\b
+)""", flags=re.X)
+
+
+def titleFixup(text):
+    """
+    Correct case in a word assuming the presence of titles/surnames,
+    including 'McDonald', 'MacNair', 'McKilroy', and cases that
+    python's title screws up such as "Smith's".
+    """
+    
+    text = text.title()
+    text = re.sub(
+        r"\b(Mc)([a-z])",
+        lambda match: match.group(1) + match.group(2).upper(),
+        text
+    )
+    text = re.sub(
+        r"\b(Mac)([bcdfgjklmnpqrstvwxyz])([a-z]{3,})",
+        lambda m: m.group(1) + m.group(2).upper() + m.group(3),
+        text
+    )
+    text = re.sub("\b(von|van|de|du|of)\b",
+        lambda m: m.group(1).lower,
+        text
+    )
+    text = re.sub(r"'S\b", "'s", text)
+    text = ''.join((text[0].upper(), text[1:]))
+    
+    return text
+
+def checkForOcrDerp(tdenv, systemName, stationName):
+    match = ocrDerp.search(stationName.upper())
+    if match:
+        tdenv.NOTE(
+            "Ignoring '{}/{}' because it looks like OCR derp."
+            .format(systemName, stationName)
+        )
+        return match
+    return None

tradedangerous/version.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# --------------------------------------------------------------------
+# Copyright (C) Oliver 'kfsone' Smith 2014 <oliver@kfs.org>:
+# Copyright (C) Bernd 'Gazelle' Gollesch 2016, 2017
+# Copyright (C) Jonathan 'eyeonus' Jones 2018, 2019
+#
+# You are free to use, redistribute, or even print and eat a copy of
+# this software so long as you include this copyright notice.
+# I guarantee there is at least one bug neither of us knew about.
+# --------------------------------------------------------------------
+
+"""just keeper of current version"""
+
+# TODO: remember to update tests when version changes
+__version__ = '12.7.6'
+