PyFT8 3.7.0__tar.gz → 3.7.3__tar.gz

{pyft8-3.7.0 → pyft8-3.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyFT8
-Version: 3.7.0
+Version: 3.7.3
 Summary: FT8 Decoding and Encoding in Python with test/loopback code
 Author-email: G1OJS <g1ojs@yahoo.com>
 License-Expression: GPL-3.0-or-later
@@ -23,7 +23,7 @@ Dynamic: license-file
 # PyFT8 [![PyPI Downloads](https://static.pepy.tech/personalized-badge/pyft8?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/pyft8) [![PyPI Downloads](https://static.pepy.tech/personalized-badge/pyft8?period=weekly&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads/wk)](https://pepy.tech/projects/pyft8) [PyPI link](https://pypi.org/project/PyFT8/)
 # All-Python FT8 Transceiver GUI / Command Line Modem
 
-This repository contains the source code for PyFT8, an all-Python open source FT8 transceiver that you can run as a basic GUI or from the command line to receive and transmit. Decoding performance (number of decodes) is about 70% of that achieved by WSJT-x in NORM mode, but (tbc) slightly above ft8_lib. 
+This repository contains the source code for PyFT8, an all-Python open source FT8 transceiver that you can run as a basic GUI or from the command line to receive and transmit. Decoding performance (number of decodes) is very close (~90% of decodes) to that of WSJT-x 2.7.0 running in FAST mode. 
 
 PyFT8 is somewhat experimental, with a focus on demonstrating FT8 written in Python, but can be used as a standalone replacement for WSJT-x and other software.
 
@@ -112,9 +112,32 @@ upload = Y
 
 ## Performance Compared with WSJT-x
 
-The image below shows the number of decodes from PyFT8, WSJT-x V2.7.0 running in NORM mode, using the same 10 minutes of busy 20m audio that is used to test ft8_lib. On this dataset, WSJT-x performs almost the same on DEEP mode as it does on NORM, and both beat PyFT8. However, PyFT8 gets very close to WSJT-x's performance on FAST mode.
+The image below shows the number of decodes from PyFT8, WSJT-x V2.7.0 running in NORM mode, using 5 minutes of busy 20m audio taken from the files used to test ft8_lib. On this dataset, WSJT-x performs almost the same on DEEP mode as it does on NORM, so the latter has been left off. Also, using AP with DEEP mode causes WSJT-x to be starved of CPU on my old PC (that's my theory), resulting in some cycles with hardly any decodes. PyFT8 gets about 90% of the decodes that WSJT-x gets on FAST mode, and about 78% of those it gets on DEEP mode.
 
-<img width="900" height="900" alt="live playback vs wsjtx_v2 7 0" src="https://github.com/user-attachments/assets/853e98b2-5b9b-4241-b5f3-181fe193642e" />
+<img width="900" height="900" alt="live wav playback vs wsjtx_v2 7 0" src="https://github.com/user-attachments/assets/d05755c9-07d2-4e2f-98ff-c15f64b320fe" />
+
+
+## Decoding approach
+The approach to decoding, as at v3.7.0, is as follows.
+
+ - As audio arrives
+    - FFT to a spectrum 4 times per symbol duration & assemble a time-frequency grid with 2 bins per tone.
+    - Collect all audio samples for later full cycle FFT
+ - Search the time frequency grid for candidates based on Costas array sync score (using middle Costas block)
+ - Send candidates for decode using symbol & tone points drawn directly from the search grid to get early decodes using limited LDPC cycles.
+
+This often decodes around three quarters of the candidates decoded by WSJT-x, with the steps below getting most of the rest:
+
+ - For candidates that haven't decoded:
+     - FFT the full cycle audio, downsample to 200 samples per second & shift the signal to f = 0
+     - Explore fine time and frequency offsets to maximise the Costas sync score
+     - Check for bad candidates (count maxima of Costas tones)
+     - Extract the LLR & wait for decoder to pick up the candidate
+ - For candidates awaiting decode from above:
+     - Try more extensive LDPC
+     - Try several 'A Priori' masks and input to more LDPC attempts, recording the output LLR on failures
+     - Try Ordered Statistics Decoding on the base LLR
+     - Try Ordered Statistics Decoding on the LLRs output from the AP+LDPC runs above
 
 ## Limitations
 PyFT8 doesn't decode / encode *all* message types. The table below shows which are handled.

{pyft8-3.7.0 → pyft8-3.7.3}/PyFT8/databases.py

@@ -2,6 +2,9 @@ import threading, os, pickle, json
 from PyFT8.pskreporter import PSKR_MQTT_listener
 from PyFT8.time_utils import time_utils
 
+
+# call hash calculator for receiver ==============================================================
+
 call_hashes, hashes_for_calls = {}, {}
 
 def add_call_hashes(call):
@@ -22,31 +25,8 @@ def add_call_hashes(call):
         call_hashes[(x, m)] = call
     hashes_for_calls[call] = hashes
 
-def grid_to_latlong(grid, centre = True):
-    lat, lon = -90, -180
-    grid = grid.upper()
-    if centre:
-        grid = grid + "LL44LL44LL44"[len(grid):]
-    mults = [20, 2, 2/24, 0.2/24, 0.2/(24*24), 0.02/(24*24)]
-    grid = grid[:2*len(mults)]
-    pairs = [grid[i:i+2] for i in range(0,len(grid),2)]
-    for i, p in enumerate(pairs):
-        zero = [ord('A'),ord('0')][i % 2]
-        lon += mults[i] * (ord(p[0]) - zero)
-        lat += mults[i] * (ord(p[1]) - zero) / 2
-    return (lat, lon)
-        
-def grids_to_dist_brg(sq1, sq2, units):
-    from numpy import sin, cos, asin, atan2, sqrt, radians, degrees
-    ll1, ll2 = grid_to_latlong(sq1), grid_to_latlong(sq2)
-    lats = [radians(ll1[0]), radians(ll2[0])]
-    dlat, dlon = radians(ll2[0] - ll1[0]), radians(ll2[1] - ll1[1])
-    s_lats, c_lats = sin(lats), cos(lats)
-    a = sin(dlat/2)**2 + c_lats[0] * c_lats[1] * sin(dlon/2)**2
-    r = 6371 * 2 * asin(sqrt(a))
-    b = atan2(c_lats[1] * sin(dlon), c_lats[0] * s_lats[1] - s_lats[0] * c_lats[1] * cos(dlon))
-    b *= (1.0 if 'km' in units else 0.621371)
-    return (r, degrees(b) % 360)
+
+# Disk-synced dictionary class for History ======================================================
 
 class DiskDict:
     def __init__(self, file, autosave_t0):
@@ -84,11 +64,12 @@ class DiskDict:
             except:
                 return
 
+# History class ==============================================================================================
+
 class History:
-    def __init__(self, config_folder, my_call, home_square = '', geo_units = 'km', pskr_refresh_mins = 20):
+    def __init__(self, config_folder, my_call, home_square, geo_units, pskr_refresh_mins = 20):
         self.pskr_refresh_mins = pskr_refresh_mins
         self.geo_units = geo_units
-        self.log_cache = None
         self.my_call = my_call
         self.home_square = home_square
         self.home_square_lev4 = home_square[:4]
@@ -104,26 +85,52 @@ class History:
         self.heard_by_me = DiskDict(f"{config_folder}/heard_by_me.json", 5)     # all-time record of heard by me
         self.call_to_grid = DiskDict(f"{config_folder}/call_to_grid.json", 7)   # all time cache call -> fine locator
         self.band_TxRx_homecall_recent_L4grid = DiskDict(f"{config_folder}/recent_l4grid.json", 9) # last 20 mins data -> per band tx/rx & current band detail
+        self.start()
+
+    def start(self):
+        mqtt = PSKR_MQTT_listener(self.home_square_lev4, self._add_mqtt_spot)
+        threading.Thread(target = self._count_activity, daemon = True).start()
+
+    
+    # external query-type functions --------------------------------------------------------
+
+    def is_in_new_alert(self, band, call, new_alert_data):
+        result = False
+        if band in new_alert_data:
+            result = call in new_alert_data[band]
+        return result
+
+    def is_hearing_me(self, band, call, in_last_n_minutes):
+        result = False
+        if band in self.hearing_me.data:
+            if call in self.hearing_me.data[band]:
+                result = int(self.hearing_me.data[band][call]['t']) > (time_utils.time() - 60*in_last_n_minutes)
+        return result        
+      
+    def get_spot_counts(self, band, call):
+        tx_reports = self.band_TxRx_homecall_recent_L4grid.data.get(f"{band}_0_{call}", [])
+        rx_reports = self.band_TxRx_homecall_recent_L4grid.data.get(f"{band}_1_{call}", [])
+        n_spotting = len(tx_reports) if tx_reports else 0
+        n_spotted = len(rx_reports) if rx_reports else 0
+        return n_spotted, n_spotting
+
+    def get_geo_text(self, call):
+        geo_text = ''
+        grid = self.call_to_grid.data.get(call, False)
+        if grid:
+            loc = grid if self.geo_units == 'grid' else self._get_dist_brg(grid, self.geo_units)
+            units_str = '' if self.geo_units == 'grid' else ('km' if 'km' in self.geo_units else 'mi')
+            geo_text = f"{int(loc[0]):5d}{units_str} {int(loc[1]):3d}°"
+        return geo_text
+
+    # external storage-like functions -------------------------------------------------------------------
 
-    def start_collect_new(self):
-        mqtt = PSKR_MQTT_listener(self.home_square_lev4, self.add_mqtt_spot)
-        threading.Thread(target = self.count_activity, daemon = True).start()
-
-    def incorporate_log_data(self, log_cache):
-        self.log_cache = log_cache
-        for key in log_cache:
-            key_parts = key.split('_')
-            if len(key_parts) > 1:
-                c, b, m = key_parts
-                if m == 'FT8':
-                    #print(f"Parsing adif log: Add hearing & heard by {c} on {b}")
-                    self._add_myspots_record(self.hearing_me.data, None, b, c, 0, 0)
-                    self._add_myspots_record(self.heard_by_me.data, None, b, c, 0, 0)
-                else:
-                    print(m)
+    def add_bidirectional_hearing_heard_spot(self, band, their_call, time_on):
+        self._add_myspots_record(self.hearing_me.data, band, their_call, time_on, 0)
+        self._add_myspots_record(self.heard_by_me.data, band, their_call, time_on, 0)
 
     def load_hearing_heard_from_all_file(self, bands):
-        recs = self.parse_all_txt()
+        recs = self._parse_all_txt()
         if not any(recs):
             return
         for b in bands:
@@ -133,79 +140,90 @@ class History:
             if r['md'] == 'FT8':
                 band = self.freqs_to_bands.get(round(r['fMHz'], 1), None)
                 if band is not None:
-                    TxRx = 'Tx' if (r['TxRx'] == 'Tx' or r['call_b'] == self.my_call) else 'Rx'
-                    call = r['call_b'] if TxRx == 'Rx' else r['call_a']
-                    if len(call) > 3:
-                        data = self.heard_by_me.data if TxRx == 'Rx' else self.hearing_me.data
-                        self._add_myspots_record(data, None, band, call, 0, 0)
+                    data = None
+                    if len(r['call_a']) > 3 and len(r['call_b']) > 3:
+                        if r['call_a'] == self.my_call:
+                            data, call = self.hearing_me.data, r['call_b']      # call b is calling me, so call b ishearing me
+                        if r['call_b'] == self.my_call and not r['call_a'].startswith('CQ'): 
+                            data, call = self.heard_by_me.data, r['call_a']     # I'm calling call a, so I'm hearing call a
+                    if data:
+                        self._add_myspots_record(data, band, call, r['t'], r['rp'])
 
-    def parse_all_txt(self):
-        rows, recs = None, []
-        if os.path.exists(self.all_file):
-            with open(self.all_file, 'r') as f:
-                rows = f.readlines()
-        if rows is not None:
-            for r in rows:
-                fields = r.strip().split()
-                if len(fields) > 8:
-                    call_a_pos = 7 if fields[7] != '~' else 8
-                    recs.append({'fMHz':float(fields[1]), 'TxRx':fields[2], 'md':fields[3],
-                                 'call_a':fields[call_a_pos], 'call_b':fields[call_a_pos + 1]} )
-        if not any(recs):
-            print("Didn't find any records in an ALL.txt file in the config folder")
-        return recs
-
-    def process_message(self, message, band_info, myCall):
+    def process_message_for_history(self, message, band_info, myCall):
         m = message
-        self._write_all_txt_row(m['cyclestart_string'], float(band_info['fMHz']), 'Rx', 'FT8',
-                                m['their_snr'], m['dt'], m['fHz'], m['hail'], m['their_call'], m['grid_rpt'])
-        self._add_myspots_record(self.heard_by_me.data, self.heard_by_me_new,
-                                band_info['current_band'], m['their_call'], int(time_utils.time()), m['their_snr'])
-        if m['hail'] == myCall: # They're calling me, so they're hearing me
-            rpt = m['grid_rpt'][-3:]
+        self._write_all_txt_row(m['all_txt_format'])
+        hail, their_call, grid_rpt = m['msg_tuple']
+        current_band = band_info['current_band']
+        self._add_myspots_record(self.heard_by_me.data, current_band,
+                                 their_call, int(time_utils.time()), m['their_snr'],
+                                 new_alert_data = self.heard_by_me_new)
+        if hail == myCall: # They're calling me, so they're hearing me
+            rpt = grid_rpt[-3:]
             if rpt.isnumeric():
-                self._add_myspots_record(self.hearing_me.data, self.hearing_me_new,
-                                       band_info['current_band'], m['their_call'], int(time_utils.time()), int(rpt))
+                self._add_myspots_record(self.hearing_me.data, current_band, their_call,
+                                         int(time_utils.time()), int(rpt), new_alert_data = self.heard_by_me_new)
+
 
+    # ----- internal storing / writing functions ----------------------------------------------------------
 
-    def _write_all_txt_row(self, cyclestart_string, fMHz, TxRx, mode, snr_str, dt, fHz, hail, their_call, grid_rpt):
+    def _write_all_txt_row(self, row_text):
         filemode = 'w' if not os.path.exists(self.all_file) else 'a'
-        row = f"{cyclestart_string} {fMHz:8.3f} {TxRx} {mode} {snr_str} {dt:4.1f} {fHz:4.0f} {hail} {their_call} {grid_rpt}"
         with open(self.all_file, filemode) as f:
-            f.write(f"{row}\n")
+            f.write(f"{row_text}\n")
 
-    def add_mqtt_spot(self, d):
+    def _add_mqtt_spot(self, d):
         tnow = int(time_utils.time())
         sc, rc = (d['sc'], d['sl']), (d['rc'], d['rl'])
         for iTxRx, call_grid in enumerate([sc, rc]):
             call, grid = call_grid
-            self.store_best_grid(call, grid)
+            self._store_best_grid(call, grid)
             if self.home_square_lev4 in grid:
-                self.add_homespots_record(f"{d['b']}_{iTxRx}_{call}", tnow)
+                self._add_homespots_record(f"{d['b']}_{iTxRx}_{call}", tnow)
         if d['sc'] == self.my_call:
-            self._add_myspots_record(self.hearing_me.data, self.hearing_me_new, d['b'], d['rc'], tnow, d['rp'])
+            self._add_myspots_record(self.hearing_me.data, d['b'], d['rc'], tnow, d['rp'], new_alert_data = self.hearing_me_new)
         if d['rc'] == self.my_call:
-            self._add_myspots_record(self.heard_by_me.data, self.heard_by_me_new, d['b'], d['sc'], tnow, d['rp'])
+            self._add_myspots_record(self.heard_by_me.data, d['b'], d['sc'], tnow, d['rp'], new_alert_data = self.heard_by_me_new)
 
-    def store_best_grid(self, call, grid):
+    def _add_homespots_record(self, key, t):
+        self.band_TxRx_homecall_recent_L4grid.data.setdefault(key, [])
+        self.band_TxRx_homecall_recent_L4grid.data[key].append(int(t))
+
+    def _store_best_grid(self, call, grid):
         if call.startswith('<'): return
         existing_grid = self.call_to_grid.data.get(call, '')
         if len(grid) > len(existing_grid):
             self.call_to_grid.data[call] = grid
-        
-    def add_homespots_record(self, key, t):
-        self.band_TxRx_homecall_recent_L4grid.data.setdefault(key, [])
-        self.band_TxRx_homecall_recent_L4grid.data[key].append(int(t))
 
-    def _add_myspots_record(self, historic_data, new_alert_data, band, call, t, rp):
-        self._update_new_alert(band, call, historic_data, new_alert_data)
+    def _parse_all_txt(self):
+        import time, calendar
+        rows, recs = None, []
+        if os.path.exists(self.all_file):
+            with open(self.all_file, 'r') as f:
+                rows = f.readlines()
+        if rows is not None:
+            for r in rows:
+                fields = r.strip().split()
+                if len(fields) > 8:
+                    call_a_pos = 7 if fields[7] != '~' else 8
+                    if len(fields[0]) == 13:
+                        time_tuple = time.strptime(fields[0], "%y%m%d_%H%M%S")
+                        t = calendar.timegm(time_tuple)
+                        recs.append({'t':t , 'fMHz':float(fields[1]), 'TxRx':fields[2], 'md':fields[3],
+                                     'rp':fields[4], 'call_a':fields[call_a_pos], 'call_b':fields[call_a_pos + 1]} )
+        if not any(recs):
+            print("Didn't find any records in an ALL.txt file in the config folder")
+        return recs
+    
+    def _add_myspots_record(self, historic_data, band, call, t, rp, new_alert_data = None):
+        self._update_new_alert_data(band, call, historic_data, new_alert_data)
         historic_data.setdefault(band, {})
         if call in historic_data[band]:
             if int(t) < int(historic_data[band][call]['t']):
                 return
-        historic_data[band][call] = {'t': int(t),'rp':int(rp)}
+        if t > 0:
+            historic_data[band][call] = {'t': int(t),'rp':int(rp)}
 
-    def _update_new_alert(self, band, call, historic_data, new_alert_data):
+    def _update_new_alert_data(self, band, call, historic_data, new_alert_data):
         if new_alert_data is None:
             return
         new = band not in historic_data
@@ -215,20 +233,14 @@ class History:
             new_alert_data.setdefault(band, [])
             new_alert_data[band].append(call)
 
-    def is_in_new_alert(self, band, call, new_alert_data):
-        result = False
-        if band in new_alert_data:
-            result = call in new_alert_data[band]
-        return result
+    def _get_dist_brg(self, grid, units):
+        self.dist_brg_cache.setdefault(grid, _grids_to_dist_brg(self.home_square, grid, units))
+        return self.dist_brg_cache[grid]
 
-    def is_hearing_me(self, band, call, since_seconds = 120):
-        result = False
-        if band in self.hearing_me.data:
-            if call in self.hearing_me.data[band]:
-                result = int(self.hearing_me.data[band][call]['t']) > since_seconds
-        return result        
-      
-    def count_activity(self):
+
+    # ---- Internal analysis functions -------------------------------------------------------------
+
+    def _count_activity(self):
         import numpy as np
         while True:
             time_utils.sleep(5)
@@ -261,33 +273,38 @@ class History:
                         if nremotes > current_winner[1]:
                             if c != self.my_call:
                                 self.home_most_remotes[b][iTxRx] = (c, nremotes)
+                                
+# Geo Helpers =================================================================
 
-    def get_spot_counts(self, band, call):
-        tx_reports = self.band_TxRx_homecall_recent_L4grid.data.get(f"{band}_0_{call}", [])
-        rx_reports = self.band_TxRx_homecall_recent_L4grid.data.get(f"{band}_1_{call}", [])
-        n_spotting = len(tx_reports) if tx_reports else 0
-        n_spotted = len(rx_reports) if rx_reports else 0
-        return n_spotted, n_spotting
+def _grids_to_dist_brg(sq1, sq2, units):
+    from numpy import sin, cos, asin, atan2, sqrt, radians, degrees
+    ll1, ll2 = _grid_to_latlong(sq1), _grid_to_latlong(sq2)
+    lats = [radians(ll1[0]), radians(ll2[0])]
+    dlat, dlon = radians(ll2[0] - ll1[0]), radians(ll2[1] - ll1[1])
+    s_lats, c_lats = sin(lats), cos(lats)
+    a = sin(dlat/2)**2 + c_lats[0] * c_lats[1] * sin(dlon/2)**2
+    r = 6371 * 2 * asin(sqrt(a))
+    b = atan2(c_lats[1] * sin(dlon), c_lats[0] * s_lats[1] - s_lats[0] * c_lats[1] * cos(dlon))
+    b *= (1.0 if 'km' in units else 0.621371)
+    return (r, degrees(b) % 360)  
 
-    def get_dist_brg(self, grid, units):
-        self.dist_brg_cache.setdefault(grid, grids_to_dist_brg(self.home_square, grid, units))
-        return self.dist_brg_cache[grid]
 
-    def get_message_extra_info(self, their_call, current_band):
-        geo_text = self.get_geo_text(their_call)
-        wb_time = self.log_cache.get(their_call,'') 
-        wb_text = f"wb: {time_utils.format_duration(time_utils.time() - float(wb_time))}" if wb_time else ''
-        hearing_me = '# ' if self.is_hearing_me(current_band, their_call) else ' '
-        return hearing_me, wb_text, geo_text 
+def _grid_to_latlong(grid, centre = True):
+    lat, lon = -90, -180
+    grid = grid.upper()
+    if centre:
+        grid = grid + "LL44LL44LL44"[len(grid):]
+    mults = [20, 2, 2/24, 0.2/24, 0.2/(24*24), 0.02/(24*24)]
+    grid = grid[:2*len(mults)]
+    pairs = [grid[i:i+2] for i in range(0,len(grid),2)]
+    for i, p in enumerate(pairs):
+        zero = [ord('A'),ord('0')][i % 2]
+        lon += mults[i] * (ord(p[0]) - zero)
+        lat += mults[i] * (ord(p[1]) - zero) / 2
+    return (lat, lon)
+        
 
-    def get_geo_text(self, call):
-        geo_text = ''
-        grid = self.call_to_grid.data.get(call, False)
-        if grid:
-            loc = grid if self.geo_units == 'grid' else self.get_dist_brg(grid, self.geo_units)
-            units_str = '' if self.geo_units == 'grid' else ('km' if 'km' in self.geo_units else 'mi')
-            geo_text = f"{int(loc[0]):5d}{units_str} {int(loc[1]):3d}°"
-        return geo_text
+# ==================ADIF LOGGING=========================================================================
                 
 def ensure_file_exists(path, header = None):
     try:
@@ -314,6 +331,10 @@ class ADIF:
         self.cache[log_dict['call']] = tm
         self.cache[cbm] = tm
 
+    def get_worked_before_info(self, their_call):
+        wb_time = self.cache.get(their_call,'') 
+        return f"wb: {time_utils.format_duration(time_utils.time() - float(wb_time))}" if wb_time else ''
+
     def _build_cache(self):
         import calendar, time
         def parse(rec, field):