tradedangerous 12.7.6__py3-none-any.whl

tradedangerous/misc/diff-system-csvs.py

@@ -0,0 +1,159 @@
+#! /usr/bin/env python
+# Small tool for comparing two system.csv files.
+# Usage:
+#  misc/compare-system-csvs.py <old file> <new file>
+# Checks for stars that changed names, stars that moved
+# and conflicts where two stars occupy the same space.
+
+# Writes output suitable for "corrections.py" to stderr,
+# so you could do something like:
+#  $ diff-system-csvs.py oldSystem.csv newSystem.csv 2>corrections.txt
+
+import sys
+
+if len(sys.argv) != 3:
+    raise SystemExit("Usage: {} <old file> <new file>".format(
+                sys.argv[0]
+            ))
+
+import csv
+import re
+from pathlib import Path
+from collections import namedtuple
+
+class Loc(namedtuple('Loc', [ 'x', 'y', 'z' ])):
+    def __str__(self):
+        return "{},{},{}".format(self.x, self.y, self.z)
+
+
+class Item(namedtuple('Item', [ 'norm', 'name', 'loc' ])):
+    pass
+
+
+normalizeRe = re.compile('[^A-Za-z0-9\' ]')
+
+
+def readFile(filename):
+    path = Path(filename)
+    if not path.exists():
+        raise SystemExit("File not found: {}".format(filename))
+    
+    names, locs = dict(), dict()
+    
+    with path.open("r", encoding="utf-8") as fh:
+        csvin = csv.reader(fh, delimiter=',', quotechar='\'', doublequote=True)
+        # skip headings
+        next(csvin)
+        
+        for line in csvin:
+            name = line[0]
+            x = float(line[1])
+            y = float(line[2])
+            z = float(line[3])
+            
+            normalized = normalizeRe.sub('', name).upper()
+            try:
+                prevEntry = names[normalized]
+            except KeyError:
+                pass
+            else:
+                print("Name clash: {}, this entry: {}, prev entry: {}".format(
+                            normalized,
+                            name,
+                            prevEntry.name
+                        ))
+            item = Item(normalized, name, Loc(x, y, z))
+            names[normalized] = item
+            if item.loc in locs:
+                print("{}: Duplicate location: {} and {} at {}".format(
+                            filename, locs[item.loc].name, name, item.loc
+                        ))
+            else:
+                locs[item.loc] = item
+    
+    return names, locs
+
+
+oldNames, oldLocs = readFile(sys.argv[1])
+newNames, newLocs = readFile(sys.argv[2])
+
+for oldName, oldItem in oldNames.items():
+    try:
+        # Look the item up in the new names dict
+        newItem = newNames[oldItem.norm]
+    except:
+        pass
+    else:
+        if oldItem.name != newItem.name:
+            if oldItem.name.upper() == oldItem.name.upper():
+                # Case changed, we can live with this.
+                print("CAUTION: {} changed to {}".format(
+                            oldItem.name,
+                            newItem.name,
+                        ))
+            else:
+                # Punctuation or something else, we need
+                # a correction.
+                print("{} became {}".format(
+                            oldItem.name, newItem.name
+                        ))
+                print("  \"{}\": \"{}\",".format(
+                            oldItem.name.upper(),
+                            newItem.name,
+                        ), file=sys.stderr)
+        
+        # Name didn't change, did the position?
+        if oldItem.loc != newItem.loc:
+            print("{} moved from {} -> {}".format(
+                        oldItem.name, oldItem.loc, newItem.loc
+                    ))
+        
+        # We don't need to do a location check on this one.
+        try:
+            del newLocs[newItem.loc]
+        except KeyError:
+            pass
+        continue
+    
+    # We didn't find the old name in the new list, check
+    # to see if there is a new star at the old position.
+    try:
+        newItem = newLocs[oldItem.loc]
+    except:
+        pass
+    else:
+        # we found something at the exact loc, which I
+        # assume means the name changed.
+        try:
+            # we know it's there, we can remove it
+            del newLocs[oldItem.loc]
+        except KeyError:
+            pass
+        print("{} ({}) changed name to {}".format(
+                    oldItem.name,
+                    oldItem.loc,
+                    newItem.name
+                ))
+        print("  \"{}\": \"{}\",".format(
+                    oldItem.name.upper(),
+                    newItem.name,
+                ), file=sys.stderr)
+        continue
+    
+    # we didn't find it, so as best we can tell it
+    # has been removed. there's no easy way for us
+    # to catch the case of a move and a minor reloc.
+    print("{} ({}) was removed".format(
+                oldItem.name,
+                oldItem.loc
+            ))
+    print("  \"{}\": DELETED,".format(
+                oldItem.name.upper(),
+            ), file=sys.stderr)
+
+
+for newLoc, newItem in newLocs.items():
+    if newLoc in oldLocs:
+        continue
+    print("{} ({}) was added".format(newItem.name, newLoc))
+

tradedangerous/misc/eddb.py

@@ -0,0 +1,81 @@
+#! /usr/bin/env python
+
+"""
+Provides a simple API for streaming data from EDDB in essentially raw format.
+
+Example:
+    import misc.eddb
+    for sysdata in misc.eddb.SystemsQuery():
+        print(sysdata)
+
+Original author: oliver@kfs.org
+"""
+
+import transfers
+
+BASE_URL = "http://eddb.io/archive/v3/"
+COMMODITIES_JSON = BASE_URL + "commodities.json"
+SYSTEMS_JSON = BASE_URL + "systems.json"
+STATIONS_EXT_JSON = BASE_URL + "stations.json"
+STATIONS_LITE_JSON = BASE_URL + "stations_lite.json"
+
+
+class EDDBQuery:
+    """
+    Base class for querying an EDDB data set and converting the
+    JSON results into an iterable stream.
+    
+    Example:
+        for entity in EDDBQuery():
+            print(entity)
+    """
+    
+    url = None      # Define in derived classes
+    
+    def __init__(self):
+        assert self.url
+        self.jsonData = transfers.get_json_data(self.url)
+    
+    def __iter__(self):
+        return iter(self.jsonData)
+
+class CommoditiesQuery(EDDBQuery):
+    """
+    Streams Commodities data from EDDB.
+    
+    Example:
+        for comm in CommoditiesQuery():
+            print(comm['name'])
+    """
+    url = COMMODITIES_JSON
+
+class SystemsQuery(EDDBQuery):
+    """
+    Streams System data from EDDB.
+    
+    Example:
+        for system in SystemsQuery():
+            print(system['name'])
+    """
+    url = SYSTEMS_JSON
+
+class StationsQuery(EDDBQuery):
+    """
+    Streams Station data from EDDB without trade data.
+    
+    Example:
+        for station in StationsQuery():
+            print(station['name'])
+    """
+    url = STATIONS_LITE_JSON
+
+class StationsExtQuery(StationsQuery):
+    """
+    Streams extended Station data from EDDB with trade data.
+    
+    Example:
+        for station in StationsExtQuery():
+            print(station['name'])
+    """
+    url = STATIONS_EXT_JSON
+

tradedangerous/misc/eddn.py

@@ -0,0 +1,349 @@
+"""
+Utilities for reading from the Elite Dangerous Data Network.
+
+Example usages:
+    
+    # Simple:
+    import eddn
+    listener = eddn.Listener()
+    while True:
+        batch = listener.get_batch()
+        if batch:
+            print("Got batch of %d" % len(batch))
+    
+    # Advanced:
+    import eddn
+    
+    listener = eddn.Listener(
+        minBatchTime=3,         # Allow at least 3-s for a batch,
+        maxBatchTime=5,         # But allow upto 5s,
+        reconnectTimeout=300,   # Reconnect after 5 minutes without data,
+        burstLimit=500,         # Drain upto 500 prices between polls,
+    )
+    
+    def handle_listener_error(e):
+        print("Listener Error:", e)
+    
+    def process_batch(batch):
+        stations = set()
+        items = set()
+        software = set()
+        for price in batch:
+            stations.add(price.station)
+            items.add(price.item)
+            software.add(price.software + ":" + price.version)
+        print("Batch: %d entries" % len(batch))
+        print("Stations: %s" % (','.join(stations)))
+        print("Items: %s" % (','.join(items)))
+    
+    print("Listening for 100 batches")
+    while listener.stats['batches'] < 100:
+        batch = listener.get_batch(onerror=handle_listener_error)
+        if batch:
+            process_batch(batch)
+        stats, errors = listener.stats, listener.errors
+        if errors or (listener.stats['batches'] % 5) == 0:
+            print("Stats:")
+            for stat in sorted(stats.keys()):
+                print("  {:<20s} {:>10n}".format(stat, stats[stat]))
+        if errors:
+            print("ERRORS:")
+            for error in sorted(errors.keys()):
+                print("  {:<20s} {:>10n}".format(error, errors[error]))
+            listener.clear_errors()
+    
+    listener.reset_counters()
+"""
+
+# Copyright (C) Oliver 'kfsone' Smith <oliver@kfs.org> 2015
+#
+# Conditional permission to copy, modify, refactor or use this
+# code is granted so long as attribution to the original author
+# is included.
+
+import json
+import time
+import zlib
+import zmq
+
+from collections import defaultdict
+from collections import namedtuple
+
+
+class MarketPrice(namedtuple('MarketPrice', [
+        'system',
+        'station',
+        'item',
+        'buy',
+        'sell',
+        'demand',
+        'supply',
+        'timestamp',
+        'uploader',
+        'software',
+        'version',
+        ])):
+    pass
+
+
+class Listener:
+    """
+    Provides an object that will listen to the Elite Dangerous Data Network
+    firehose and capture messages for later consumption.
+    
+    Rather than individual updates, prices are captured across a window of
+    between minBatchTime and maxBatchTime. When a new update is received,
+    Rather than returning individual messages, messages are captured across
+    a window of potentially several seconds and returned to the caller in
+    batches.
+    
+    Attributes:
+        zmqContext          Context this object is associated with,
+        minBatchTime        Allow at least this long for a batch (ms),
+        maxBatchTime        Don't allow a batch to run longer than this (ms),
+        reconnectTimeout    Reconnect the socket after this long with no data,
+        burstLimit          Read a maximum of this many messages between
+                            timer checks
+        
+        subscriber          ZMQ socket we're using
+        stats               Counters of nominal events
+        errors              Counters of off-nominal events
+        lastRecv            time of the last receive (or 0)
+    """
+    
+    uri = 'tcp://eddn-relay.elite-markets.net:9500'
+    supportedSchema = 'http://schemas.elite-markets.net/eddn/commodity/1'
+    
+    def __init__(
+        self,
+        zmqContext=None,
+        minBatchTime=5.,    # seconds
+        maxBatchTime=10.,   # seconds
+        reconnectTimeout=180.,  # seconds
+        burstLimit=200,
+    ):
+        assert burstLimit > 0
+        if not zmqContext:
+            zmqContext = zmq.Context()
+        self.zmqContext = zmqContext
+        self.subscriber = None
+        
+        self.minBatchTime = minBatchTime
+        self.maxBatchTime = maxBatchTime
+        self.reconnectTimeout = reconnectTimeout
+        self.burstLimit = burstLimit
+        
+        self.reset_counters()
+        self.connect()
+
+
+    def connect(self):
+        """
+        Start a connection
+        """
+        # tear up the new connection first
+        if self.subscriber:
+            self.subscriber.close()
+            del self.subscriber
+        self.subscriber = newsub = self.zmqContext.socket(zmq.SUB)
+        newsub.setsockopt(zmq.SUBSCRIBE, b"")
+        newsub.connect(self.uri)
+        self.lastRecv = time.time()
+        self.lastJsData = None
+
+
+    def disconnect(self):
+        del self.subscriber
+
+
+    def clear_errors(self):
+        self.errors = defaultdict(int)
+
+
+    def reset_counters(self):
+        self.clear_errors()
+        self.stats = defaultdict(int)
+
+
+    def wait_for_data(self, softCutoff, hardCutoff):
+        """
+        Waits for data until maxBatchTime ms has elapsed
+        or cutoff (absolute time) has been reached.
+        """
+        
+        now = time.time()
+        
+        cutoff = min(softCutoff, hardCutoff)
+        if self.lastRecv < now - self.reconnectTimeout:
+            if self.lastRecv:
+                self.errors['reconnects'] += 1
+            self.connect()
+            now = time.time()
+        
+        nextCutoff = min(now + self.minBatchTime, cutoff)
+        if now > nextCutoff:
+            return False
+        
+        timeout = (nextCutoff - now) * 1000     # milliseconds
+        
+        # Wait for an event
+        events = self.subscriber.poll(timeout=timeout)
+        if events == 0:
+            return False
+        return True
+
+
+    def get_batch(self, onerror=None):
+        """
+        Greedily collect deduped prices from the firehose over a
+        period of between minBatchTime and maxBatchTime, with
+        built-in auto-reconnection if there is nothing from the
+        firehose for a period of time.
+        
+        As json data is decoded, it is stored in self.lastJsData.
+        
+        Parameters:
+            onerror
+                None or a function/lambda that takes an error
+                string and deals with it.
+        
+        Returns:
+            A list of MarketPrice entries based on the data read.
+            Prices are deduped per System+Station+Item, so that
+            if two entries are received for the same combination,
+            only the most recent with the newest timestamp is kept.
+        
+        Errors:
+            Errors are acculumated in the .errors dictionary. If you
+            supply an 'onerror' function they are also passed to it.
+        """
+        now = time.time()
+        hardCutoff = now + self.maxBatchTime
+        softCutoff = now + self.minBatchTime
+        
+        # hoists
+        supportedSchema = self.supportedSchema
+        sub = self.subscriber
+        stats, errors = self.stats, self.errors
+        
+        # Prices are stored as a dictionary of
+        # (sys,stn,item) => [MarketPrice]
+        # The list thing is a trick to save us having to do
+        # the dictionary lookup twice.
+        batch = defaultdict(list)
+        
+        while self.wait_for_data(softCutoff, hardCutoff):
+            # When wait_for_data returns True, there is some data waiting,
+            # possibly multiple messages. At this point we can afford to
+            # suck down whatever is waiting in "nonblocking" mode until
+            # we reach the burst limit or we get EAGAIN.
+            bursts = 0
+            for _ in range(self.burstLimit):
+                self.lastJsData = None
+                try:
+                    zdata = sub.recv(flags=zmq.NOBLOCK, copy=False)
+                    stats['recvs'] += 1
+                except zmq.error.Again:
+                    break
+                
+                bursts += 1
+                
+                try:
+                    jsdata = zlib.decompress(zdata)
+                except Exception as e:
+                    errors['deflate'] += 1
+                    if onerror:
+                        onerror("zlib.decompress: %s: %s"%(type(e), e))
+                    continue
+                
+                bdata = jsdata.decode()
+                
+                try:
+                    data = json.loads(bdata)
+                except ValueError as e:
+                    errors['loads'] += 1
+                    if onerror:
+                        onerror("json.loads: %s: %s"%(type(e), e))
+                    continue
+                
+                self.lastJsData = jsdata
+                
+                try:
+                    schema = data["$schemaRef"]
+                except KeyError:
+                    errors['schemaref'] += 1
+                    if onerror:
+                        onerror("missing schema ref")
+                    continue
+                if schema != supportedSchema:
+                    errors['schema'] += 1
+                    if onerror:
+                        onerror("unsupported schema: "+schema)
+                    continue
+                try:
+                    header = data["header"]
+                    message = data["message"]
+                    system = message["systemName"].upper()
+                    station = message["stationName"].upper()
+                    item = message["itemName"].upper()
+                    buy = int(message["buyPrice"])
+                    sell = int(message["sellPrice"])
+                    demand = message["demand"]
+                    supply = message["stationStock"]
+                    timestamp = message["timestamp"]
+                    uploader = header["uploaderID"]
+                    software = header["softwareName"]
+                    swVersion = header["softwareVersion"]
+                except (KeyError, ValueError) as e:
+                    errors['json'] += 1
+                    if onerror:
+                        onerror("invalid json: %s: %s"%(type(e), e))
+                    continue
+                
+                # We've received real data.
+                stats['prices'] += 1
+                
+                # Normalize timestamps
+                timestamp = timestamp.replace("T"," ").replace("+00:00","")
+                
+                # We'll get either an empty list or a list containing
+                # a MarketPrice. This saves us having to do the expensive
+                # index operation twice.
+                oldEntryList = batch[(system, station, item)]
+                if oldEntryList:
+                    if oldEntryList[0].timestamp > timestamp:
+                        stats['timeseq'] += 1
+                        continue
+                    stats['timeseq']
+                else:
+                    # Add a blank entry to make the list size > 0
+                    oldEntryList.append(None)
+                
+                # Here we're replacing the contents of the list.
+                # This simple array lookup is several hundred times less
+                # expensive than looking up a potentially large dictionary
+                # by STATION/SYSTEM:ITEM...
+                oldEntryList[0] = MarketPrice(
+                    system, station, item,
+                    buy, sell,
+                    demand, supply,
+                    timestamp,
+                    uploader, software, swVersion,
+                )
+            
+            # For the edge-case where we wait 4.999 seconds and then
+            # get a burst of data: stick around a little longer.
+            if bursts >= self.burstLimit:
+                stats['numburst'] += 1
+                stats['maxburst'] = max(stats['maxburst'], bursts)
+                softCutoff = min(softCutoff, time.time() + 0.5)
+        
+        # to get average batch length, divide batchlen/batches.
+        # you could do the same with prices/batches except that
+        stats['batches'] += 1
+        if not batch:
+            stats['emptybatches'] += 1
+        else:
+            stats['batchlen'] += len(batch)
+        
+        return [ entry[0] for entry in batch.values() ]