callhash 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- callhash/__init__.py +61 -0
- callhash/_nhash.py +122 -0
- callhash/parse.py +132 -0
- callhash/table.py +465 -0
- callhash-1.1.0.dist-info/METADATA +134 -0
- callhash-1.1.0.dist-info/RECORD +9 -0
- callhash-1.1.0.dist-info/WHEEL +5 -0
- callhash-1.1.0.dist-info/licenses/LICENSE +21 -0
- callhash-1.1.0.dist-info/top_level.txt +1 -0
callhash/__init__.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""WSJT-X callsign-hash mechanism — Bob Jenkins lookup3 (seed 146)
|
|
2
|
+
plus an accumulator that resolves hashes from ``<call>`` announcement
|
|
3
|
+
markers in decoded text.
|
|
4
|
+
|
|
5
|
+
Background. WSJT-X's FT4/FT8/WSPR packet formats compress compound
|
|
6
|
+
callsigns (e.g. ``K1ABC/QRP``, ``VE3/K1ABC``) into 22-bit (FT8/FT4)
|
|
7
|
+
or 15-bit (WSPR) hashes; receivers maintain a session table that
|
|
8
|
+
maps the hash back to plaintext when the call has been observed in a
|
|
9
|
+
``<call>`` first-occurrence message. Both ``jt9`` and ``wsprd`` use
|
|
10
|
+
this same hash function (``nhash`` / Bob Jenkins lookup3, seed 146).
|
|
11
|
+
|
|
12
|
+
Per-slot decoder invocations (psk-recorder, wsprdaemon-client) start
|
|
13
|
+
with an empty session table, so most hashed packets emerge as
|
|
14
|
+
``<...>`` placeholders. This module reconstructs the table at the
|
|
15
|
+
consumer side from the same announcement markers.
|
|
16
|
+
|
|
17
|
+
Public API:
|
|
18
|
+
|
|
19
|
+
from callhash import CallHashTable, hash22, hash15, nhash
|
|
20
|
+
|
|
21
|
+
table = CallHashTable.load_or_new("/var/lib/<client>/hashtable.json")
|
|
22
|
+
table.observe("260507 1234 -12 +0.45 1250 ` <K1ABC/QRP> CQ FT8")
|
|
23
|
+
table.by_hash22(hash22("K1ABC/QRP")) # → "K1ABC/QRP"
|
|
24
|
+
clean = table.normalise_brackets("<K1ABC/QRP>") # → "K1ABC/QRP"
|
|
25
|
+
clean = table.normalise_brackets("<...>") # → None
|
|
26
|
+
table.save() # persists to JSON
|
|
27
|
+
|
|
28
|
+
The hash function is bit-exact against WSJT-X's canonical
|
|
29
|
+
``lib/wsprcode/nhash.c`` for all 23 reference vectors in
|
|
30
|
+
``tests/test_nhash.py`` (including 11/12/13-byte boundary cases that
|
|
31
|
+
exercise distinct final-block branches).
|
|
32
|
+
"""
|
|
33
|
+
from ._nhash import (
|
|
34
|
+
MASK10,
|
|
35
|
+
MASK12,
|
|
36
|
+
MASK15,
|
|
37
|
+
MASK22,
|
|
38
|
+
WSJTX_INITVAL,
|
|
39
|
+
hash10,
|
|
40
|
+
hash12,
|
|
41
|
+
hash15,
|
|
42
|
+
hash22,
|
|
43
|
+
nhash,
|
|
44
|
+
)
|
|
45
|
+
from .parse import parse_message
|
|
46
|
+
from .table import CallHashTable
|
|
47
|
+
|
|
48
|
+
__all__ = [
|
|
49
|
+
"CallHashTable",
|
|
50
|
+
"MASK10",
|
|
51
|
+
"MASK12",
|
|
52
|
+
"MASK15",
|
|
53
|
+
"MASK22",
|
|
54
|
+
"WSJTX_INITVAL",
|
|
55
|
+
"hash10",
|
|
56
|
+
"hash12",
|
|
57
|
+
"hash15",
|
|
58
|
+
"hash22",
|
|
59
|
+
"nhash",
|
|
60
|
+
"parse_message",
|
|
61
|
+
]
|
callhash/_nhash.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Bob Jenkins lookup3 ``nhash`` — Python port of WSJT-X's
|
|
2
|
+
``lib/wsprd/nhash.c``.
|
|
3
|
+
|
|
4
|
+
WSJT-X uses ``nhash(call, len(call), initval=146)`` as the basis for
|
|
5
|
+
the 22-bit / 15-bit / 12-bit / 10-bit hash values that compress
|
|
6
|
+
compound callsigns into FT4/FT8/WSPR packet payloads. Both
|
|
7
|
+
``decode_ft8``-style and ``wsprd``-style decoders use the same
|
|
8
|
+
function with the same seed (146) — see
|
|
9
|
+
``/tmp/wsjtx-mirror/lib/wsprd/wsprd_utils.c:int unpk_(...)`` for the
|
|
10
|
+
WSPR-side call site and ``/tmp/wsjtx-mirror/lib/hash.f90`` for the
|
|
11
|
+
FT8/FT4 side.
|
|
12
|
+
|
|
13
|
+
This port matches the C output bit-exactly for ASCII inputs.
|
|
14
|
+
Cross-validation vectors are in ``tests/test_callhash_nhash.py``,
|
|
15
|
+
generated by compiling the upstream C and capturing its outputs.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
WSJTX_INITVAL = 146 # Seed used by both jt9 and wsprd.
|
|
20
|
+
MASK22 = 0x3FFFFF # FT8/FT4 compound-call hash width.
|
|
21
|
+
MASK15 = 0x7FFF # WSPR Type 3 hash width.
|
|
22
|
+
MASK12 = 0xFFF # ARRL contest hash variant.
|
|
23
|
+
MASK10 = 0x3FF # Smallest WSJT-X hash variant.
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _rot(x: int, k: int) -> int:
|
|
27
|
+
"""32-bit rotate-left."""
|
|
28
|
+
return ((x << k) | (x >> (32 - k))) & 0xFFFFFFFF
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _mix(a: int, b: int, c: int) -> tuple[int, int, int]:
|
|
32
|
+
"""The mix() macro from nhash.c — six rotate/xor/sub/add rounds."""
|
|
33
|
+
a = (a - c) & 0xFFFFFFFF; a ^= _rot(c, 4); c = (c + b) & 0xFFFFFFFF
|
|
34
|
+
b = (b - a) & 0xFFFFFFFF; b ^= _rot(a, 6); a = (a + c) & 0xFFFFFFFF
|
|
35
|
+
c = (c - b) & 0xFFFFFFFF; c ^= _rot(b, 8); b = (b + a) & 0xFFFFFFFF
|
|
36
|
+
a = (a - c) & 0xFFFFFFFF; a ^= _rot(c, 16); c = (c + b) & 0xFFFFFFFF
|
|
37
|
+
b = (b - a) & 0xFFFFFFFF; b ^= _rot(a, 19); a = (a + c) & 0xFFFFFFFF
|
|
38
|
+
c = (c - b) & 0xFFFFFFFF; c ^= _rot(b, 4); b = (b + a) & 0xFFFFFFFF
|
|
39
|
+
return a, b, c
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _final(a: int, b: int, c: int) -> int:
|
|
43
|
+
"""The final() macro from nhash.c — seven rotate/xor/sub rounds."""
|
|
44
|
+
c ^= b; c = (c - _rot(b, 14)) & 0xFFFFFFFF
|
|
45
|
+
a ^= c; a = (a - _rot(c, 11)) & 0xFFFFFFFF
|
|
46
|
+
b ^= a; b = (b - _rot(a, 25)) & 0xFFFFFFFF
|
|
47
|
+
c ^= b; c = (c - _rot(b, 16)) & 0xFFFFFFFF
|
|
48
|
+
a ^= c; a = (a - _rot(c, 4)) & 0xFFFFFFFF
|
|
49
|
+
b ^= a; b = (b - _rot(a, 14)) & 0xFFFFFFFF
|
|
50
|
+
c ^= b; c = (c - _rot(b, 24)) & 0xFFFFFFFF
|
|
51
|
+
return c
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def nhash(key: bytes | str, initval: int = WSJTX_INITVAL) -> int:
|
|
55
|
+
"""Compute Bob Jenkins lookup3 hash of ``key`` with the given seed.
|
|
56
|
+
|
|
57
|
+
Matches the C output of WSJT-X's ``nhash()`` for ASCII inputs.
|
|
58
|
+
Returns a 32-bit unsigned integer; callers mask down to the bit
|
|
59
|
+
width their protocol uses (see :func:`hash22`, :func:`hash15`).
|
|
60
|
+
|
|
61
|
+
``key`` may be ``bytes`` or ``str``; strings are encoded as ASCII
|
|
62
|
+
(matching the C call sites that hash Fortran character arrays
|
|
63
|
+
containing callsign text).
|
|
64
|
+
"""
|
|
65
|
+
if isinstance(key, str):
|
|
66
|
+
key = key.encode("ascii")
|
|
67
|
+
length = len(key)
|
|
68
|
+
a = b = c = (0xDEADBEEF + length + initval) & 0xFFFFFFFF
|
|
69
|
+
|
|
70
|
+
# Process complete 12-byte blocks as three little-endian uint32s.
|
|
71
|
+
pos = 0
|
|
72
|
+
remaining = length
|
|
73
|
+
while remaining > 12:
|
|
74
|
+
a = (a + int.from_bytes(key[pos : pos + 4], "little")) & 0xFFFFFFFF
|
|
75
|
+
b = (b + int.from_bytes(key[pos + 4 : pos + 8], "little")) & 0xFFFFFFFF
|
|
76
|
+
c = (c + int.from_bytes(key[pos + 8 : pos + 12], "little")) & 0xFFFFFFFF
|
|
77
|
+
a, b, c = _mix(a, b, c)
|
|
78
|
+
remaining -= 12
|
|
79
|
+
pos += 12
|
|
80
|
+
|
|
81
|
+
# Final partial block — emulate the C switch statement, masking
|
|
82
|
+
# so the output matches the C's "read-past-end-then-mask" trick
|
|
83
|
+
# (we just don't read past the end). Zero-length input returns
|
|
84
|
+
# the initial c verbatim.
|
|
85
|
+
if remaining == 0:
|
|
86
|
+
return c
|
|
87
|
+
|
|
88
|
+
tail = key[pos : pos + remaining]
|
|
89
|
+
if remaining >= 9:
|
|
90
|
+
a = (a + int.from_bytes(tail[0:4], "little")) & 0xFFFFFFFF
|
|
91
|
+
b = (b + int.from_bytes(tail[4:8], "little")) & 0xFFFFFFFF
|
|
92
|
+
c_val = int.from_bytes(tail[8:remaining], "little")
|
|
93
|
+
c = (c + c_val) & 0xFFFFFFFF
|
|
94
|
+
elif remaining >= 5:
|
|
95
|
+
a = (a + int.from_bytes(tail[0:4], "little")) & 0xFFFFFFFF
|
|
96
|
+
b_val = int.from_bytes(tail[4:remaining], "little")
|
|
97
|
+
b = (b + b_val) & 0xFFFFFFFF
|
|
98
|
+
else:
|
|
99
|
+
a_val = int.from_bytes(tail[:remaining], "little")
|
|
100
|
+
a = (a + a_val) & 0xFFFFFFFF
|
|
101
|
+
|
|
102
|
+
return _final(a, b, c)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def hash22(call: str) -> int:
|
|
106
|
+
"""22-bit hash of ``call`` — used by FT8/FT4 compound-callsign packets."""
|
|
107
|
+
return nhash(call) & MASK22
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def hash15(call: str) -> int:
|
|
111
|
+
"""15-bit hash of ``call`` — used by WSPR Type 3 packets."""
|
|
112
|
+
return nhash(call) & MASK15
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def hash12(call: str) -> int:
|
|
116
|
+
"""12-bit hash variant — used by some ARRL-contest message types."""
|
|
117
|
+
return nhash(call) & MASK12
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def hash10(call: str) -> int:
|
|
121
|
+
"""10-bit hash variant — narrowest WSJT-X hash."""
|
|
122
|
+
return nhash(call) & MASK10
|
callhash/parse.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Shared WSJT-X message-field parser.
|
|
2
|
+
|
|
3
|
+
FT8 / FT4 / MSK144 decoders (``decode_ft8``, ``jt9``) emit a freeform
|
|
4
|
+
message body whose grammar is identical across the modes. Every
|
|
5
|
+
recorder consumer — psk-recorder, meteor-scatter, wspr-recorder — needs
|
|
6
|
+
the same best-effort extraction of tx/rx call, grid, and report, *and*
|
|
7
|
+
the same compound-callsign hash substitution via
|
|
8
|
+
:class:`~callhash.table.CallHashTable`.
|
|
9
|
+
|
|
10
|
+
Before this module the extraction was copy-pasted byte-for-byte into
|
|
11
|
+
each recorder's ``ch_tailer.py`` and the table was built but never read,
|
|
12
|
+
so hashed spots lost their call. Centralising it here means one
|
|
13
|
+
implementation that resolves hashes the same way everywhere — the whole
|
|
14
|
+
point of a shared callhash library.
|
|
15
|
+
|
|
16
|
+
``parse_message`` is intentionally lossy: the raw (hash-resolved)
|
|
17
|
+
message string is always returned in ``message`` so the caller can
|
|
18
|
+
preserve it verbatim; the parsed call/grid/report fields are best-effort.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
from typing import Any, Dict, Optional
|
|
24
|
+
|
|
25
|
+
# Standard callsigns + WSJT-X compound forms:
|
|
26
|
+
# * standard ITU call: K1ABC, AC0G, JA1AAA
|
|
27
|
+
# * suffix-form compound: K1ABC/QRP, K1ABC/MM
|
|
28
|
+
# * prefix-form compound (region/portable): VE3/K1ABC, G/K1ABC, KH6/AC0G
|
|
29
|
+
_CALL_RE = re.compile(
|
|
30
|
+
r"^"
|
|
31
|
+
r"(?:[A-Z0-9]{1,3}/)?" # optional prefix (e.g. "VE3/", "G/")
|
|
32
|
+
r"[A-Z0-9]{1,3}[0-9][A-Z0-9]{0,4}" # standard call body (XX[X][D][YY[Y][Y]])
|
|
33
|
+
r"(?:/[A-Z0-9]{1,4})?" # optional suffix (e.g. "/QRP", "/MM")
|
|
34
|
+
r"$"
|
|
35
|
+
)
|
|
36
|
+
# Maidenhead 6-char form has uppercase field+square but lowercase
|
|
37
|
+
# subsquare (per IARU convention). Tolerate either case for robustness.
|
|
38
|
+
_GRID_RE = re.compile(r"^[A-R]{2}[0-9]{2}(?:[A-Xa-x]{2})?$")
|
|
39
|
+
_REPORT_RE = re.compile(r"^R?([+-]?\d+)$")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _is_grid(token: str) -> bool:
|
|
43
|
+
"""A Maidenhead locator, but never the sign-off ``RR73``. RR73 has a
|
|
44
|
+
locator's shape (a square in the Arctic Ocean), and WSJT-X never
|
|
45
|
+
sends that square: a grid read from it would place every station
|
|
46
|
+
that signs off near the pole."""
|
|
47
|
+
return bool(_GRID_RE.match(token)) and token.upper() != "RR73"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _plain(token: str) -> Optional[str]:
|
|
51
|
+
"""Return a bare callsign token, or ``None`` if it's still bracketed.
|
|
52
|
+
|
|
53
|
+
``parse_message`` resolves the whole message first, so any token that
|
|
54
|
+
*still* carries angle brackets here (``<...>`` or a numeric hash we've
|
|
55
|
+
never seen announced) is genuinely unrecoverable — drop it from the
|
|
56
|
+
call fields rather than emitting a placeholder.
|
|
57
|
+
"""
|
|
58
|
+
if not token or token.startswith("<"):
|
|
59
|
+
return None
|
|
60
|
+
return token
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_message(message: str, table: Optional[Any] = None) -> Dict[str, Any]:
|
|
64
|
+
"""Parse a decoded FT8/FT4/MSK144 message body.
|
|
65
|
+
|
|
66
|
+
When ``table`` (a :class:`CallHashTable`) is supplied, the message is
|
|
67
|
+
first run through :meth:`CallHashTable.resolve_message` so bracketed
|
|
68
|
+
hashes that we've learned are substituted with plaintext calls — this
|
|
69
|
+
is what stops us shipping ``<...>`` spots whose call we actually know.
|
|
70
|
+
|
|
71
|
+
Recognized shapes (all approximate; freeform messages return empties):
|
|
72
|
+
"CQ <tx_call> [<grid>]" — directed CQ
|
|
73
|
+
"<rx_call> <tx_call> <grid>" — first contact w/ grid
|
|
74
|
+
"<rx_call> <tx_call> R <grid>" — reply that acknowledges, w/ grid
|
|
75
|
+
"<rx_call> <tx_call> [R]<report>" — signal report
|
|
76
|
+
"<rx_call> <tx_call> [73|RR73]" — close
|
|
77
|
+
|
|
78
|
+
Returns a dict with keys ``message`` (the hash-resolved text — callers
|
|
79
|
+
should store THIS, not the raw input), ``tx_call``, ``rx_call``,
|
|
80
|
+
``grid`` (all ``""`` when absent) and ``report`` (``None`` when absent).
|
|
81
|
+
"""
|
|
82
|
+
if table is not None:
|
|
83
|
+
message = table.resolve_message(message)
|
|
84
|
+
|
|
85
|
+
out: Dict[str, Any] = {
|
|
86
|
+
"message": message,
|
|
87
|
+
"tx_call": "",
|
|
88
|
+
"rx_call": "",
|
|
89
|
+
"grid": "",
|
|
90
|
+
"report": None,
|
|
91
|
+
}
|
|
92
|
+
tokens = message.split()
|
|
93
|
+
if not tokens:
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
if tokens[0] == "CQ":
|
|
97
|
+
# "CQ [target] <tx_call> [grid]" — `target` may be a region tag
|
|
98
|
+
# like "DX", "EU", "POTA" that isn't a callsign. Scan past
|
|
99
|
+
# non-call tokens until the first call-shaped one (the sender),
|
|
100
|
+
# then look for a grid in the remaining tokens.
|
|
101
|
+
for i, tok in enumerate(tokens[1:], start=1):
|
|
102
|
+
candidate = _plain(tok)
|
|
103
|
+
if candidate is not None and _CALL_RE.match(candidate):
|
|
104
|
+
out["tx_call"] = candidate
|
|
105
|
+
for later in tokens[i + 1:]:
|
|
106
|
+
if _is_grid(later):
|
|
107
|
+
out["grid"] = later
|
|
108
|
+
break
|
|
109
|
+
break
|
|
110
|
+
else:
|
|
111
|
+
# <rx_call> <tx_call> [grid|R grid|report|RR73|73]
|
|
112
|
+
rx_candidate = _plain(tokens[0])
|
|
113
|
+
if rx_candidate is not None and _CALL_RE.match(rx_candidate):
|
|
114
|
+
out["rx_call"] = rx_candidate
|
|
115
|
+
if len(tokens) >= 2:
|
|
116
|
+
tx_candidate = _plain(tokens[1])
|
|
117
|
+
if tx_candidate is not None and _CALL_RE.match(tx_candidate):
|
|
118
|
+
out["tx_call"] = tx_candidate
|
|
119
|
+
if len(tokens) >= 3:
|
|
120
|
+
tail = tokens[2]
|
|
121
|
+
if tail == "R" and len(tokens) >= 4 and _is_grid(tokens[3]):
|
|
122
|
+
out["grid"] = tokens[3]
|
|
123
|
+
elif _is_grid(tail):
|
|
124
|
+
out["grid"] = tail
|
|
125
|
+
else:
|
|
126
|
+
m = _REPORT_RE.match(tail)
|
|
127
|
+
if m:
|
|
128
|
+
try:
|
|
129
|
+
out["report"] = int(m.group(1))
|
|
130
|
+
except ValueError:
|
|
131
|
+
pass
|
|
132
|
+
return out
|
callhash/table.py
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
"""``CallHashTable`` — accumulator that maps WSJT-X callsign hashes
|
|
2
|
+
back to plaintext from ``<call>`` announcement markers.
|
|
3
|
+
|
|
4
|
+
Two consumers (psk-recorder, wsprdaemon-client) and the same
|
|
5
|
+
mechanism: when a compound callsign is first transmitted, both ends
|
|
6
|
+
broadcast it in plaintext between angle brackets — ``<K1ABC/QRP>``;
|
|
7
|
+
subsequent packets use a 22-bit (FT8/FT4) or 15-bit (WSPR) hash, and
|
|
8
|
+
each side resolves the hash via its in-memory table. Per-invocation
|
|
9
|
+
decoders (jt9, wsprd) start with an empty table, so most hashed
|
|
10
|
+
packets surface as the literal placeholder ``<...>``.
|
|
11
|
+
|
|
12
|
+
This class reconstructs the table at the consumer side by watching
|
|
13
|
+
decoded text for ``<call>`` markers — the announcement is itself a
|
|
14
|
+
plaintext sighting we can hash and store. Persisting the table to
|
|
15
|
+
JSON lets the resolution survive daemon restarts so the cumulative
|
|
16
|
+
mapping grows over time.
|
|
17
|
+
|
|
18
|
+
The class deliberately stays small and stateless beyond the in-memory
|
|
19
|
+
maps so it can be embedded in either client without dragging in an
|
|
20
|
+
import graph.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import re
|
|
27
|
+
import threading
|
|
28
|
+
from datetime import datetime, timezone
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Callable, Dict, Iterable, Optional, Tuple
|
|
31
|
+
|
|
32
|
+
from ._nhash import hash12, hash15, hash22
|
|
33
|
+
|
|
34
|
+
log = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# Plaintext compound-callsign markers in decoded text:
|
|
38
|
+
# - <K1ABC/QRP> announcement (canonical case — two- or three-segment
|
|
39
|
+
# call separated by '/').
|
|
40
|
+
# - <K1ABC> a hashed call already resolved by the decoder
|
|
41
|
+
# from its session table; reading us the call back.
|
|
42
|
+
# - <...> literal "unresolved hash" placeholder (no useful
|
|
43
|
+
# content).
|
|
44
|
+
# The regex below matches the first two but not the literal ellipsis
|
|
45
|
+
# placeholder.
|
|
46
|
+
_BRACKET_CALL_RE = re.compile(r"<([A-Z0-9][A-Z0-9/]{1,15})>")
|
|
47
|
+
_LITERAL_UNRESOLVED = "<...>"
|
|
48
|
+
# A bracketed all-numeric token is a hash the decoder couldn't resolve
|
|
49
|
+
# from its own session table — jt9 ``-Y`` (FST4W / MSK144) and the
|
|
50
|
+
# patched ``decode_ft8`` (FT8 / FT4) both emit the 22-bit hash this way
|
|
51
|
+
# (``<NNNNNNN>``, up to 7 digits since MAX22 = 4194304). No decoder we
|
|
52
|
+
# drive emits the narrower 12-bit form as a number, so a numeric bracket
|
|
53
|
+
# token is always a 22-bit hash.
|
|
54
|
+
_BRACKET_HASH_RE = re.compile(r"<(\d{1,7})>")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class CallHashTable:
|
|
58
|
+
"""Per-(client, radiod) cache mapping WSJT-X hashes → plaintext call.
|
|
59
|
+
|
|
60
|
+
Thread-safe in the only way that matters here: ``observe`` and
|
|
61
|
+
``by_hash*`` may be called concurrently by a tailer thread + a
|
|
62
|
+
parser thread without external locking.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
SCHEMA_VERSION = 1 # bump when the on-disk JSON shape changes
|
|
66
|
+
|
|
67
|
+
def __init__(self, source_path: Optional[Path] = None) -> None:
|
|
68
|
+
self._source_path: Optional[Path] = (
|
|
69
|
+
Path(source_path) if source_path is not None else None
|
|
70
|
+
)
|
|
71
|
+
self._lock = threading.Lock()
|
|
72
|
+
# Two parallel maps (one per protocol). Same plaintext call
|
|
73
|
+
# is in both — the cost of double-storage is trivial for the
|
|
74
|
+
# observed table sizes (10²-10³ entries) and avoids per-lookup
|
|
75
|
+
# bit-mask choices on the hot path.
|
|
76
|
+
self._by_h22: Dict[int, str] = {}
|
|
77
|
+
self._by_h15: Dict[int, str] = {}
|
|
78
|
+
# 12-bit map kept for API symmetry with the other widths. No
|
|
79
|
+
# decoder we drive emits a 12-bit hash as a resolvable number
|
|
80
|
+
# (decode_ft8's 12-bit site stays ``<...>``), so this is rarely
|
|
81
|
+
# hit in practice, but populating it is free.
|
|
82
|
+
self._by_h12: Dict[int, str] = {}
|
|
83
|
+
# Collision guard. A hash slot claimed by 2+ DISTINCT calls is
|
|
84
|
+
# ambiguous — a reverse lookup can't tell which call the hash
|
|
85
|
+
# meant, so we refuse to resolve it rather than risk emitting the
|
|
86
|
+
# wrong callsign into a spot (a wrong call pollutes wsprnet /
|
|
87
|
+
# pskreporter; a missing call is benign). This matters because
|
|
88
|
+
# OUR table is persistent and accumulates 10²–10³+ calls, so it
|
|
89
|
+
# sees far more collisions than WSJT-X's short-lived per-session
|
|
90
|
+
# table — especially in the 15-bit space (32 768 slots). These
|
|
91
|
+
# sets are recomputed on load, not persisted.
|
|
92
|
+
self._collided_h22: set[int] = set()
|
|
93
|
+
self._collided_h15: set[int] = set()
|
|
94
|
+
self._collided_h12: set[int] = set()
|
|
95
|
+
# First-seen timestamp per call — useful for operator forensics
|
|
96
|
+
# ("when did this compound call first appear?") and bounded
|
|
97
|
+
# eviction in future versions.
|
|
98
|
+
self._first_seen: Dict[str, str] = {}
|
|
99
|
+
self._observations = 0 # total announcements seen (for stats)
|
|
100
|
+
self._dirty = False # save() can short-circuit when no change
|
|
101
|
+
|
|
102
|
+
# ----- observation -----
|
|
103
|
+
|
|
104
|
+
def observe(self, text: str) -> int:
|
|
105
|
+
"""Scan ``text`` for ``<call>`` markers and add each to the table.
|
|
106
|
+
|
|
107
|
+
Returns the number of NEW entries inserted by this call.
|
|
108
|
+
Multiple calls in one line are all extracted; existing entries
|
|
109
|
+
are left unchanged (announcements are stable per call).
|
|
110
|
+
"""
|
|
111
|
+
added = 0
|
|
112
|
+
with self._lock:
|
|
113
|
+
for match in _BRACKET_CALL_RE.findall(text):
|
|
114
|
+
call = match.strip()
|
|
115
|
+
if not call:
|
|
116
|
+
continue
|
|
117
|
+
if not _looks_like_callsign(call):
|
|
118
|
+
continue
|
|
119
|
+
self._observations += 1
|
|
120
|
+
if self._add_locked(call):
|
|
121
|
+
added += 1
|
|
122
|
+
if added:
|
|
123
|
+
log.debug("callhash: +%d new call(s); table now %d (h22) / %d (h15)",
|
|
124
|
+
added, len(self._by_h22), len(self._by_h15))
|
|
125
|
+
return added
|
|
126
|
+
|
|
127
|
+
def add(self, call: str) -> bool:
|
|
128
|
+
"""Direct-add a single plaintext callsign. Returns True if new."""
|
|
129
|
+
with self._lock:
|
|
130
|
+
return self._add_locked(call)
|
|
131
|
+
|
|
132
|
+
def ingest_calls(self, calls: Iterable[str]) -> int:
|
|
133
|
+
"""Add a batch of plaintext callsigns. Returns the count of NEW
|
|
134
|
+
entries.
|
|
135
|
+
|
|
136
|
+
Convenience for consumers that discover full calls in decoded
|
|
137
|
+
output (rather than ``<call>`` brackets) — e.g. wspr-recorder's
|
|
138
|
+
per-cycle spot ingest. Non-callsign-shaped entries are skipped.
|
|
139
|
+
"""
|
|
140
|
+
added = 0
|
|
141
|
+
with self._lock:
|
|
142
|
+
for call in calls:
|
|
143
|
+
if not call:
|
|
144
|
+
continue
|
|
145
|
+
call = call.strip()
|
|
146
|
+
if not _looks_like_callsign(call):
|
|
147
|
+
continue
|
|
148
|
+
if self._add_locked(call):
|
|
149
|
+
added += 1
|
|
150
|
+
return added
|
|
151
|
+
|
|
152
|
+
def _add_locked(self, call: str) -> bool:
|
|
153
|
+
is_new = call not in self._first_seen
|
|
154
|
+
self._index_call(call)
|
|
155
|
+
if is_new:
|
|
156
|
+
self._first_seen[call] = datetime.now(tz=timezone.utc).isoformat()
|
|
157
|
+
self._dirty = True
|
|
158
|
+
return is_new
|
|
159
|
+
|
|
160
|
+
def _index_call(self, call: str) -> None:
|
|
161
|
+
"""Insert ``call`` into the three width-keyed maps, flagging any
|
|
162
|
+
slot already owned by a *different* call as collided.
|
|
163
|
+
|
|
164
|
+
The first call to claim a slot keeps it; a second, distinct call
|
|
165
|
+
that hashes to the same slot makes it ambiguous (added to the
|
|
166
|
+
collided set) so neither resolves. Detection is idempotent and
|
|
167
|
+
order-independent, so live observation and on-disk replay
|
|
168
|
+
produce the same ambiguous-slot set.
|
|
169
|
+
"""
|
|
170
|
+
for hfn, fwd, collided in (
|
|
171
|
+
(hash22, self._by_h22, self._collided_h22),
|
|
172
|
+
(hash15, self._by_h15, self._collided_h15),
|
|
173
|
+
(hash12, self._by_h12, self._collided_h12),
|
|
174
|
+
):
|
|
175
|
+
h = hfn(call)
|
|
176
|
+
existing = fwd.get(h)
|
|
177
|
+
if existing is None:
|
|
178
|
+
fwd[h] = call
|
|
179
|
+
elif existing != call:
|
|
180
|
+
collided.add(h)
|
|
181
|
+
|
|
182
|
+
# ----- lookup -----
|
|
183
|
+
|
|
184
|
+
def by_hash22(self, h: int) -> Optional[str]:
|
|
185
|
+
"""Resolve a 22-bit hash (FT8/FT4) to plaintext, or None.
|
|
186
|
+
|
|
187
|
+
Returns None for an ambiguous (collided) slot — see the collision
|
|
188
|
+
guard in ``__init__`` — so we never return a guessed call.
|
|
189
|
+
"""
|
|
190
|
+
with self._lock:
|
|
191
|
+
h &= 0x3FFFFF
|
|
192
|
+
if h in self._collided_h22:
|
|
193
|
+
return None
|
|
194
|
+
return self._by_h22.get(h)
|
|
195
|
+
|
|
196
|
+
def by_hash15(self, h: int) -> Optional[str]:
|
|
197
|
+
"""Resolve a 15-bit hash (WSPR Type 3) to plaintext, or None.
|
|
198
|
+
|
|
199
|
+
Returns None for an ambiguous (collided) slot.
|
|
200
|
+
"""
|
|
201
|
+
with self._lock:
|
|
202
|
+
h &= 0x7FFF
|
|
203
|
+
if h in self._collided_h15:
|
|
204
|
+
return None
|
|
205
|
+
return self._by_h15.get(h)
|
|
206
|
+
|
|
207
|
+
def by_hash12(self, h: int) -> Optional[str]:
|
|
208
|
+
"""Resolve a 12-bit hash to plaintext, or None.
|
|
209
|
+
|
|
210
|
+
Returns None for an ambiguous (collided) slot.
|
|
211
|
+
"""
|
|
212
|
+
with self._lock:
|
|
213
|
+
h &= 0xFFF
|
|
214
|
+
if h in self._collided_h12:
|
|
215
|
+
return None
|
|
216
|
+
return self._by_h12.get(h)
|
|
217
|
+
|
|
218
|
+
# ----- token / message resolution -----
|
|
219
|
+
|
|
220
|
+
def resolve_token(self, token: str) -> Optional[str]:
|
|
221
|
+
"""Resolve a single decoded-message token to a plaintext call.
|
|
222
|
+
|
|
223
|
+
This is the canonical, shared substitution primitive used by
|
|
224
|
+
every consumer (psk / meteor-scatter / wspr) so they behave
|
|
225
|
+
identically. Cases:
|
|
226
|
+
|
|
227
|
+
* ``<NNNNNNN>`` (numeric hash) → :meth:`by_hash22` lookup;
|
|
228
|
+
returns the plaintext call if we've seen it announced, else
|
|
229
|
+
``None`` (still genuinely unknown — keep the placeholder).
|
|
230
|
+
* ``<...>`` → ``None`` (decoder discarded the hash number;
|
|
231
|
+
nothing to look up).
|
|
232
|
+
* ``<CALL>`` / ``<CALL/QRP>`` → the decoder resolved it from its
|
|
233
|
+
own session table; strip the brackets, **seed our table** with
|
|
234
|
+
the sighting, and return the bare call.
|
|
235
|
+
* bare token (``K1ABC``, ``CQ``, ``EM38``) → returned unchanged.
|
|
236
|
+
* anything else in brackets (garbage) → ``None``.
|
|
237
|
+
"""
|
|
238
|
+
if not token:
|
|
239
|
+
return token
|
|
240
|
+
t = token.strip()
|
|
241
|
+
if not (t.startswith("<") and t.endswith(">") and len(t) > 2):
|
|
242
|
+
return t # bare token
|
|
243
|
+
if t == _LITERAL_UNRESOLVED:
|
|
244
|
+
return None
|
|
245
|
+
m = _BRACKET_HASH_RE.fullmatch(t)
|
|
246
|
+
if m:
|
|
247
|
+
return self.by_hash22(int(m.group(1)))
|
|
248
|
+
m = _BRACKET_CALL_RE.fullmatch(t)
|
|
249
|
+
if m:
|
|
250
|
+
call = m.group(1)
|
|
251
|
+
if _looks_like_callsign(call):
|
|
252
|
+
self.add(call) # self-seeding sighting
|
|
253
|
+
return call
|
|
254
|
+
return None # bracketed garbage
|
|
255
|
+
|
|
256
|
+
def resolve_message(self, message: str) -> str:
|
|
257
|
+
"""Return ``message`` with every resolvable hash token replaced by
|
|
258
|
+
its plaintext call.
|
|
259
|
+
|
|
260
|
+
Unresolvable tokens (``<...>`` or a numeric hash we've never seen
|
|
261
|
+
announced) are left exactly as-is, so the raw text is preserved
|
|
262
|
+
when we genuinely can't recover the call. Token spacing is
|
|
263
|
+
normalised to single spaces.
|
|
264
|
+
"""
|
|
265
|
+
if not message:
|
|
266
|
+
return message
|
|
267
|
+
out = []
|
|
268
|
+
for tok in message.split():
|
|
269
|
+
resolved = self.resolve_token(tok)
|
|
270
|
+
out.append(resolved if resolved is not None else tok)
|
|
271
|
+
return " ".join(out)
|
|
272
|
+
|
|
273
|
+
def __contains__(self, call: str) -> bool:
|
|
274
|
+
with self._lock:
|
|
275
|
+
return call in self._first_seen
|
|
276
|
+
|
|
277
|
+
def __len__(self) -> int:
|
|
278
|
+
with self._lock:
|
|
279
|
+
return len(self._first_seen)
|
|
280
|
+
|
|
281
|
+
@property
|
|
282
|
+
def observations(self) -> int:
|
|
283
|
+
"""Total ``<call>`` markers observed (incl. duplicates)."""
|
|
284
|
+
with self._lock:
|
|
285
|
+
return self._observations
|
|
286
|
+
|
|
287
|
+
# ----- normalisation helper -----
|
|
288
|
+
|
|
289
|
+
@staticmethod
|
|
290
|
+
def normalise_brackets(token: str) -> Optional[str]:
|
|
291
|
+
"""Canonicalise a possibly-bracketed callsign token.
|
|
292
|
+
|
|
293
|
+
Per the wsprd / jt9 output convention:
|
|
294
|
+
* ``<K1ABC>`` or ``<K1ABC/QRP>`` → strip brackets, return
|
|
295
|
+
the inner call (it has been resolved by the decoder).
|
|
296
|
+
* ``<...>`` → return ``None`` (unresolved hash; no useful
|
|
297
|
+
information at this point — caller may try
|
|
298
|
+
:meth:`by_hash22` / :meth:`by_hash15` if it has a hash to
|
|
299
|
+
look up).
|
|
300
|
+
* Anything else → return as-is.
|
|
301
|
+
"""
|
|
302
|
+
if not token:
|
|
303
|
+
return token
|
|
304
|
+
t = token.strip()
|
|
305
|
+
if t == _LITERAL_UNRESOLVED:
|
|
306
|
+
return None
|
|
307
|
+
m = _BRACKET_CALL_RE.fullmatch(t)
|
|
308
|
+
if m:
|
|
309
|
+
return m.group(1)
|
|
310
|
+
return t
|
|
311
|
+
|
|
312
|
+
# ----- persistence -----
|
|
313
|
+
|
|
314
|
+
@classmethod
|
|
315
|
+
def load_or_new(cls, source_path: Path | str) -> "CallHashTable":
|
|
316
|
+
"""Load from JSON if it exists, else return a fresh table.
|
|
317
|
+
|
|
318
|
+
The file may be absent, empty, or contain malformed JSON
|
|
319
|
+
(cleared by an operator, half-written by a previous crashed
|
|
320
|
+
save) — in any of those cases we start fresh and log a
|
|
321
|
+
warning. We never delete the operator's file on load failure.
|
|
322
|
+
"""
|
|
323
|
+
path = Path(source_path)
|
|
324
|
+
instance = cls(source_path=path)
|
|
325
|
+
if not path.exists():
|
|
326
|
+
return instance
|
|
327
|
+
try:
|
|
328
|
+
data = json.loads(path.read_text())
|
|
329
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
330
|
+
log.warning("callhash: ignoring unreadable %s (%s); starting fresh",
|
|
331
|
+
path, e)
|
|
332
|
+
return instance
|
|
333
|
+
if not isinstance(data, dict):
|
|
334
|
+
return instance
|
|
335
|
+
if data.get("schema_version") != cls.SCHEMA_VERSION:
|
|
336
|
+
log.warning("callhash: %s has schema_version=%r, expected %d; "
|
|
337
|
+
"starting fresh", path,
|
|
338
|
+
data.get("schema_version"), cls.SCHEMA_VERSION)
|
|
339
|
+
return instance
|
|
340
|
+
for call, first_seen in (data.get("calls") or {}).items():
|
|
341
|
+
if not _looks_like_callsign(call):
|
|
342
|
+
continue
|
|
343
|
+
instance._first_seen[call] = first_seen
|
|
344
|
+
instance._index_call(call)
|
|
345
|
+
instance._observations = int(data.get("observations", 0))
|
|
346
|
+
instance._dirty = False
|
|
347
|
+
log.debug("callhash: loaded %d calls from %s", len(instance), path)
|
|
348
|
+
return instance
|
|
349
|
+
|
|
350
|
+
def save(self, path: Optional[Path | str] = None) -> None:
|
|
351
|
+
"""Atomically persist the table to disk.
|
|
352
|
+
|
|
353
|
+
Writes to ``<path>.tmp`` then ``os.replace`` so a partial
|
|
354
|
+
write can't corrupt an existing good file. No-op when no
|
|
355
|
+
observations have changed since the last save (or load).
|
|
356
|
+
"""
|
|
357
|
+
target = Path(path) if path is not None else self._source_path
|
|
358
|
+
if target is None:
|
|
359
|
+
raise ValueError(
|
|
360
|
+
"save(): no path supplied and no source_path set on table"
|
|
361
|
+
)
|
|
362
|
+
with self._lock:
|
|
363
|
+
if not self._dirty:
|
|
364
|
+
return
|
|
365
|
+
payload = {
|
|
366
|
+
"schema_version": self.SCHEMA_VERSION,
|
|
367
|
+
"saved_at": datetime.now(tz=timezone.utc).isoformat(),
|
|
368
|
+
"observations": self._observations,
|
|
369
|
+
"calls": dict(self._first_seen),
|
|
370
|
+
}
|
|
371
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
372
|
+
tmp = target.with_suffix(target.suffix + ".tmp")
|
|
373
|
+
tmp.write_text(json.dumps(payload, indent=2, sort_keys=True))
|
|
374
|
+
tmp.replace(target)
|
|
375
|
+
with self._lock:
|
|
376
|
+
self._dirty = False
|
|
377
|
+
|
|
378
|
+
# ----- decoder-seed export -----
|
|
379
|
+
|
|
380
|
+
def write_wsprd_hashtable(
|
|
381
|
+
self,
|
|
382
|
+
path: Path | str,
|
|
383
|
+
*,
|
|
384
|
+
exclude: Optional[Callable[[str], bool]] = None,
|
|
385
|
+
) -> int:
|
|
386
|
+
"""Write wsprd's ``hashtable.txt`` (15-bit index → call).
|
|
387
|
+
|
|
388
|
+
Pre-populating this file lets ``wsprd`` resolve Type-3 hashes it
|
|
389
|
+
would otherwise emit as ``<...>`` (wsprd discards the number, so
|
|
390
|
+
the only way to recover those calls is to seed the decoder ahead
|
|
391
|
+
of the run). Format matches wsprd: ``"%5d %s\\n"``.
|
|
392
|
+
|
|
393
|
+
``exclude`` is an optional predicate — calls for which it returns
|
|
394
|
+
True are omitted. wsprd-recorder passes its wsprnet
|
|
395
|
+
negative-cache filter here so consistently-rejected compounds
|
|
396
|
+
stop being re-emitted; the library itself stays oblivious to that
|
|
397
|
+
policy. Returns the number of entries written.
|
|
398
|
+
"""
|
|
399
|
+
path = Path(path)
|
|
400
|
+
with self._lock:
|
|
401
|
+
calls = list(self._first_seen)
|
|
402
|
+
count = 0
|
|
403
|
+
with open(path, "w") as f:
|
|
404
|
+
for call in calls:
|
|
405
|
+
if exclude is not None and exclude(call):
|
|
406
|
+
continue
|
|
407
|
+
f.write(f"{hash15(call):5d} {call}\n")
|
|
408
|
+
count += 1
|
|
409
|
+
return count
|
|
410
|
+
|
|
411
|
+
def write_jt9_calls(
|
|
412
|
+
self,
|
|
413
|
+
path: Path | str,
|
|
414
|
+
grids: Optional[Dict[str, str]] = None,
|
|
415
|
+
*,
|
|
416
|
+
exclude: Optional[Callable[[str], bool]] = None,
|
|
417
|
+
) -> int:
|
|
418
|
+
"""Write jt9's ``fst4w_calls.txt`` (call + grid per line).
|
|
419
|
+
|
|
420
|
+
Same role as :meth:`write_wsprd_hashtable` for the jt9 / FST4W
|
|
421
|
+
side. The library stores no grids, so the caller may pass a
|
|
422
|
+
``{call: grid}`` map; missing grids are rendered as four spaces
|
|
423
|
+
(jt9's blank-grid convention). ``exclude`` works as above.
|
|
424
|
+
Returns the number of entries written.
|
|
425
|
+
"""
|
|
426
|
+
path = Path(path)
|
|
427
|
+
grids = grids or {}
|
|
428
|
+
with self._lock:
|
|
429
|
+
calls = list(self._first_seen)
|
|
430
|
+
count = 0
|
|
431
|
+
with open(path, "w") as f:
|
|
432
|
+
for call in calls:
|
|
433
|
+
if exclude is not None and exclude(call):
|
|
434
|
+
continue
|
|
435
|
+
grid = grids.get(call) or " "
|
|
436
|
+
f.write(f"{call} {grid}\n")
|
|
437
|
+
count += 1
|
|
438
|
+
return count
|
|
439
|
+
|
|
440
|
+
# ----- introspection -----
|
|
441
|
+
|
|
442
|
+
def stats(self) -> Tuple[int, int, int]:
|
|
443
|
+
"""Return ``(unique_calls, observations, h22_entries)`` snapshot."""
|
|
444
|
+
with self._lock:
|
|
445
|
+
return len(self._first_seen), self._observations, len(self._by_h22)
|
|
446
|
+
|
|
447
|
+
def calls(self) -> list[str]:
|
|
448
|
+
"""Return a snapshot list of known plaintext calls."""
|
|
449
|
+
with self._lock:
|
|
450
|
+
return sorted(self._first_seen)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
# ── helpers ────────────────────────────────────────────────────────────────
|
|
454
|
+
|
|
455
|
+
# Loose callsign sanity check — both standard ITU calls and compound
|
|
456
|
+
# variants we care about. We let through anything that's mostly
|
|
457
|
+
# alphanumerics with at most a few '/' separators; the decoder's own
|
|
458
|
+
# bracket markers already filter out garbage upstream.
|
|
459
|
+
_CALLSIGN_VALIDATOR = re.compile(
|
|
460
|
+
r"^[A-Z0-9](?:[A-Z0-9/]{0,15})$"
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _looks_like_callsign(call: str) -> bool:
|
|
465
|
+
return bool(_CALLSIGN_VALIDATOR.fullmatch(call))
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: callhash
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: WSJT-X callsign-hash mechanism (Bob Jenkins lookup3, seed 146) plus a per-station accumulator that resolves hashes from <call> announcement markers in decoded jt9 / wsprd output.
|
|
5
|
+
Author: Rob Robinett AI6VN
|
|
6
|
+
Author-email: Michael Hauan AC0G <ac0g@hauan.org>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/HamSCI/callhash
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/HamSCI/callhash/issues
|
|
10
|
+
Keywords: wsjt-x,ham radio,ft8,ft4,wspr,callsign,hash
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# callhash
|
|
19
|
+
|
|
20
|
+
Pure-Python WSJT-X compound-callsign hash resolution.
|
|
21
|
+
|
|
22
|
+
WSJT-X's FT4/FT8/WSPR packet formats compress compound callsigns
|
|
23
|
+
(e.g. `K1ABC/QRP`, `VE3/K1ABC`) into 22-bit (FT8/FT4) or 15-bit
|
|
24
|
+
(WSPR Type 3) hashes. Receivers maintain a session table that maps
|
|
25
|
+
the hash back to plaintext when the call has been observed in a
|
|
26
|
+
`<call>` first-occurrence message. Both `jt9` and `wsprd` use the
|
|
27
|
+
same hash function — Bob Jenkins lookup3, seed 146.
|
|
28
|
+
|
|
29
|
+
When a per-slot decoder invocation starts with an empty session
|
|
30
|
+
table (the typical case for `psk-recorder`, `meteor-scatter`,
|
|
31
|
+
`wspr-recorder`, or any consumer that drives the decoder one cycle
|
|
32
|
+
at a time), most hashed packets surface as the literal `<...>`
|
|
33
|
+
placeholder — or, when the decoder is told to emit the number
|
|
34
|
+
(`jt9 -Y`, the patched `decode_ft8`), as `<NNNNNNN>`. This library
|
|
35
|
+
reconstructs the table on the consumer side from the same
|
|
36
|
+
announcement markers WSJT-X uses, and substitutes the numeric
|
|
37
|
+
hashes back to plaintext.
|
|
38
|
+
|
|
39
|
+
All three HamSCI recorders use this one library for build
|
|
40
|
+
(`observe`), lookup (`by_hashNN`), and substitution
|
|
41
|
+
(`resolve_token` / `resolve_message` / `parse_message`) — the same
|
|
42
|
+
mechanism everywhere, so a compound call learned on one mode/band
|
|
43
|
+
resolves hashes on another. Every mode hashes with the identical
|
|
44
|
+
`nhash`; only the width differs (22-bit FT8/FT4/MSK144/FST4W,
|
|
45
|
+
15-bit WSPR-2), and the table keys each width separately.
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
pip install callhash
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or from a sibling checkout (HamSCI deployment pattern):
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
pip install -e /opt/git/sigmond/callhash
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Usage
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from callhash import CallHashTable, hash22, hash15, nhash
|
|
63
|
+
|
|
64
|
+
# Persistent per-station cache.
|
|
65
|
+
table = CallHashTable.load_or_new("/var/lib/myclient/callhash.json")
|
|
66
|
+
|
|
67
|
+
# Feed any text containing <call> markers; the table extracts them.
|
|
68
|
+
table.observe("260507 1234 -12 +0.45 1250 ` <K1ABC/QRP> CQ FT8")
|
|
69
|
+
|
|
70
|
+
# Look a hash up if you happen to have one.
|
|
71
|
+
table.by_hash22(hash22("K1ABC/QRP")) # → "K1ABC/QRP"
|
|
72
|
+
table.by_hash15(hash15("K1ABC/QRP")) # → "K1ABC/QRP"
|
|
73
|
+
|
|
74
|
+
# Resolve a decoded message: substitutes <NNNNNNN>/<CALL>, drops <...>.
|
|
75
|
+
table.resolve_message("AC0G <2288505> R-12") # → "AC0G K9AN R-12" (if learned)
|
|
76
|
+
|
|
77
|
+
# Or parse a whole message into call fields, resolving hashes via the table.
|
|
78
|
+
from callhash import parse_message
|
|
79
|
+
parse_message("AC0G <2288505> R-12", table=table)
|
|
80
|
+
# → {"message": "AC0G K9AN R-12", "rx_call": "AC0G", "tx_call": "K9AN",
|
|
81
|
+
# "grid": "", "report": -12}
|
|
82
|
+
|
|
83
|
+
# Persist for the next invocation.
|
|
84
|
+
table.save()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Public API
|
|
88
|
+
|
|
89
|
+
| Symbol | Purpose |
|
|
90
|
+
| --------------------- | ---------------------------------------------------------------------- |
|
|
91
|
+
| `nhash(key, initval)` | Bob Jenkins lookup3 32-bit hash. `initval=146` matches WSJT-X. |
|
|
92
|
+
| `hash22(call)` | Convenience: `nhash(call) & 0x3FFFFF` (FT8/FT4 compound-call width). |
|
|
93
|
+
| `hash15(call)` | Convenience: `nhash(call) & 0x7FFF` (WSPR Type 3 width). |
|
|
94
|
+
| `hash12(call)` | Convenience: 12-bit mask (some ARRL contest variants). |
|
|
95
|
+
| `hash10(call)` | Convenience: 10-bit mask (narrowest WSJT-X variant). |
|
|
96
|
+
| `CallHashTable` | Persistent accumulator + lookup + substitution + collision guard. |
|
|
97
|
+
| `parse_message(msg, table=None)` | Shared WSJT-X message-field parser (tx/rx/grid/report) + hash substitution. |
|
|
98
|
+
|
|
99
|
+
`CallHashTable` resolves with `resolve_token` / `resolve_message`,
|
|
100
|
+
seeds decoders with `write_wsprd_hashtable` / `write_jt9_calls`
|
|
101
|
+
(both take an `exclude=` predicate), and **guards collisions**: a
|
|
102
|
+
hash slot claimed by two distinct calls is ambiguous, so
|
|
103
|
+
`by_hashNN` returns `None` rather than a guessed (wrong) call. This
|
|
104
|
+
matters because a persistent table accumulates far more calls than
|
|
105
|
+
WSJT-X's per-session table, so collisions — especially in the
|
|
106
|
+
15-bit space — are common.
|
|
107
|
+
|
|
108
|
+
## Correctness
|
|
109
|
+
|
|
110
|
+
`nhash` is bit-exact against WSJT-X's canonical
|
|
111
|
+
`lib/wsprcode/nhash.c` (the unmasked variant — note that
|
|
112
|
+
`lib/wsprd/nhash.c` has a 15-bit mask baked into its return). 23
|
|
113
|
+
reference vectors verified, including 11/12/13-byte boundary cases
|
|
114
|
+
that exercise distinct final-block branches in the C code.
|
|
115
|
+
|
|
116
|
+
The `CallHashTable` covers `<call>` announcement extraction,
|
|
117
|
+
hash lookups, token/message substitution, the collision guard,
|
|
118
|
+
atomic JSON persistence (write-tempfile + rename), corrupt-JSON /
|
|
119
|
+
schema-mismatch recovery, and concurrent-observe / concurrent-lookup
|
|
120
|
+
safety. 89 tests total; no live ClickHouse / WSJT-X server required
|
|
121
|
+
to run them.
|
|
122
|
+
|
|
123
|
+
## Why this lives in its own repo
|
|
124
|
+
|
|
125
|
+
The hash function and bracket-resolution logic are WSJT-X's, not
|
|
126
|
+
sigmond's. They're useful for any FT8/FT4/WSPR consumer — the
|
|
127
|
+
sigmond client suite is the primary user today, but a future
|
|
128
|
+
`hs-uploader` library or any independent log analyser will
|
|
129
|
+
benefit equally. Pure stdlib, no runtime deps, ~200 lines of
|
|
130
|
+
code; small enough to live cleanly on its own.
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
MIT.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
callhash/__init__.py,sha256=vcJ-jBhpE3D4ynmPlXb8JNfJnQRumyDmlvrz1vKiKRY,1959
|
|
2
|
+
callhash/_nhash.py,sha256=lqSYeByvDgb6ZkSIzHmwQHxwhxcRvYUe5MMpZix_vmY,4841
|
|
3
|
+
callhash/parse.py,sha256=yUHnifLF-XxnvCbdd9vxIrEkCQEGNB6PfxmQP6bx5_E,5573
|
|
4
|
+
callhash/table.py,sha256=4v79wU7etRzhSMn4uYnh2yI5pQGnFc7ZvNM35iTl-rE,18731
|
|
5
|
+
callhash-1.1.0.dist-info/licenses/LICENSE,sha256=8rBDzvlFRUZM6znjnevy0o2nzZfY4ChmNGAgTHEcl_c,1105
|
|
6
|
+
callhash-1.1.0.dist-info/METADATA,sha256=Im0FZRCVDR4xNhm7yjhL7fgOzVGmhrybg8OWl5o0mPU,5681
|
|
7
|
+
callhash-1.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
8
|
+
callhash-1.1.0.dist-info/top_level.txt,sha256=xo0toeXrc8UE0olLDsrlz-oo7xAISHra1X405Tn4wWI,9
|
|
9
|
+
callhash-1.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 HamSCI / Michael Hauan AC0G / Rob Robinett AI6VN
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
callhash
|