callhash 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
callhash/__init__.py ADDED
@@ -0,0 +1,61 @@
1
+ """WSJT-X callsign-hash mechanism — Bob Jenkins lookup3 (seed 146)
2
+ plus an accumulator that resolves hashes from ``<call>`` announcement
3
+ markers in decoded text.
4
+
5
+ Background. WSJT-X's FT4/FT8/WSPR packet formats compress compound
6
+ callsigns (e.g. ``K1ABC/QRP``, ``VE3/K1ABC``) into 22-bit (FT8/FT4)
7
+ or 15-bit (WSPR) hashes; receivers maintain a session table that
8
+ maps the hash back to plaintext when the call has been observed in a
9
+ ``<call>`` first-occurrence message. Both ``jt9`` and ``wsprd`` use
10
+ this same hash function (``nhash`` / Bob Jenkins lookup3, seed 146).
11
+
12
+ Per-slot decoder invocations (psk-recorder, wsprdaemon-client) start
13
+ with an empty session table, so most hashed packets emerge as
14
+ ``<...>`` placeholders. This module reconstructs the table at the
15
+ consumer side from the same announcement markers.
16
+
17
+ Public API:
18
+
19
+ from callhash import CallHashTable, hash22, hash15, nhash
20
+
21
+ table = CallHashTable.load_or_new("/var/lib/<client>/hashtable.json")
22
+ table.observe("260507 1234 -12 +0.45 1250 ` <K1ABC/QRP> CQ FT8")
23
+ table.by_hash22(hash22("K1ABC/QRP")) # → "K1ABC/QRP"
24
+ clean = table.normalise_brackets("<K1ABC/QRP>") # → "K1ABC/QRP"
25
+ clean = table.normalise_brackets("<...>") # → None
26
+ table.save() # persists to JSON
27
+
28
+ The hash function is bit-exact against WSJT-X's canonical
29
+ ``lib/wsprcode/nhash.c`` for all 23 reference vectors in
30
+ ``tests/test_nhash.py`` (including 11/12/13-byte boundary cases that
31
+ exercise distinct final-block branches).
32
+ """
33
+ from ._nhash import (
34
+ MASK10,
35
+ MASK12,
36
+ MASK15,
37
+ MASK22,
38
+ WSJTX_INITVAL,
39
+ hash10,
40
+ hash12,
41
+ hash15,
42
+ hash22,
43
+ nhash,
44
+ )
45
+ from .parse import parse_message
46
+ from .table import CallHashTable
47
+
48
+ __all__ = [
49
+ "CallHashTable",
50
+ "MASK10",
51
+ "MASK12",
52
+ "MASK15",
53
+ "MASK22",
54
+ "WSJTX_INITVAL",
55
+ "hash10",
56
+ "hash12",
57
+ "hash15",
58
+ "hash22",
59
+ "nhash",
60
+ "parse_message",
61
+ ]
callhash/_nhash.py ADDED
@@ -0,0 +1,122 @@
1
+ """Bob Jenkins lookup3 ``nhash`` — Python port of WSJT-X's
2
+ ``lib/wsprd/nhash.c``.
3
+
4
+ WSJT-X uses ``nhash(call, len(call), initval=146)`` as the basis for
5
+ the 22-bit / 15-bit / 12-bit / 10-bit hash values that compress
6
+ compound callsigns into FT4/FT8/WSPR packet payloads. Both
7
+ ``decode_ft8``-style and ``wsprd``-style decoders use the same
8
+ function with the same seed (146) — see
9
+ ``/tmp/wsjtx-mirror/lib/wsprd/wsprd_utils.c:int unpk_(...)`` for the
10
+ WSPR-side call site and ``/tmp/wsjtx-mirror/lib/hash.f90`` for the
11
+ FT8/FT4 side.
12
+
13
+ This port matches the C output bit-exactly for ASCII inputs.
14
+ Cross-validation vectors are in ``tests/test_callhash_nhash.py``,
15
+ generated by compiling the upstream C and capturing its outputs.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ WSJTX_INITVAL = 146 # Seed used by both jt9 and wsprd.
20
+ MASK22 = 0x3FFFFF # FT8/FT4 compound-call hash width.
21
+ MASK15 = 0x7FFF # WSPR Type 3 hash width.
22
+ MASK12 = 0xFFF # ARRL contest hash variant.
23
+ MASK10 = 0x3FF # Smallest WSJT-X hash variant.
24
+
25
+
26
+ def _rot(x: int, k: int) -> int:
27
+ """32-bit rotate-left."""
28
+ return ((x << k) | (x >> (32 - k))) & 0xFFFFFFFF
29
+
30
+
31
+ def _mix(a: int, b: int, c: int) -> tuple[int, int, int]:
32
+ """The mix() macro from nhash.c — six rotate/xor/sub/add rounds."""
33
+ a = (a - c) & 0xFFFFFFFF; a ^= _rot(c, 4); c = (c + b) & 0xFFFFFFFF
34
+ b = (b - a) & 0xFFFFFFFF; b ^= _rot(a, 6); a = (a + c) & 0xFFFFFFFF
35
+ c = (c - b) & 0xFFFFFFFF; c ^= _rot(b, 8); b = (b + a) & 0xFFFFFFFF
36
+ a = (a - c) & 0xFFFFFFFF; a ^= _rot(c, 16); c = (c + b) & 0xFFFFFFFF
37
+ b = (b - a) & 0xFFFFFFFF; b ^= _rot(a, 19); a = (a + c) & 0xFFFFFFFF
38
+ c = (c - b) & 0xFFFFFFFF; c ^= _rot(b, 4); b = (b + a) & 0xFFFFFFFF
39
+ return a, b, c
40
+
41
+
42
+ def _final(a: int, b: int, c: int) -> int:
43
+ """The final() macro from nhash.c — seven rotate/xor/sub rounds."""
44
+ c ^= b; c = (c - _rot(b, 14)) & 0xFFFFFFFF
45
+ a ^= c; a = (a - _rot(c, 11)) & 0xFFFFFFFF
46
+ b ^= a; b = (b - _rot(a, 25)) & 0xFFFFFFFF
47
+ c ^= b; c = (c - _rot(b, 16)) & 0xFFFFFFFF
48
+ a ^= c; a = (a - _rot(c, 4)) & 0xFFFFFFFF
49
+ b ^= a; b = (b - _rot(a, 14)) & 0xFFFFFFFF
50
+ c ^= b; c = (c - _rot(b, 24)) & 0xFFFFFFFF
51
+ return c
52
+
53
+
54
+ def nhash(key: bytes | str, initval: int = WSJTX_INITVAL) -> int:
55
+ """Compute Bob Jenkins lookup3 hash of ``key`` with the given seed.
56
+
57
+ Matches the C output of WSJT-X's ``nhash()`` for ASCII inputs.
58
+ Returns a 32-bit unsigned integer; callers mask down to the bit
59
+ width their protocol uses (see :func:`hash22`, :func:`hash15`).
60
+
61
+ ``key`` may be ``bytes`` or ``str``; strings are encoded as ASCII
62
+ (matching the C call sites that hash Fortran character arrays
63
+ containing callsign text).
64
+ """
65
+ if isinstance(key, str):
66
+ key = key.encode("ascii")
67
+ length = len(key)
68
+ a = b = c = (0xDEADBEEF + length + initval) & 0xFFFFFFFF
69
+
70
+ # Process complete 12-byte blocks as three little-endian uint32s.
71
+ pos = 0
72
+ remaining = length
73
+ while remaining > 12:
74
+ a = (a + int.from_bytes(key[pos : pos + 4], "little")) & 0xFFFFFFFF
75
+ b = (b + int.from_bytes(key[pos + 4 : pos + 8], "little")) & 0xFFFFFFFF
76
+ c = (c + int.from_bytes(key[pos + 8 : pos + 12], "little")) & 0xFFFFFFFF
77
+ a, b, c = _mix(a, b, c)
78
+ remaining -= 12
79
+ pos += 12
80
+
81
+ # Final partial block — emulate the C switch statement, masking
82
+ # so the output matches the C's "read-past-end-then-mask" trick
83
+ # (we just don't read past the end). Zero-length input returns
84
+ # the initial c verbatim.
85
+ if remaining == 0:
86
+ return c
87
+
88
+ tail = key[pos : pos + remaining]
89
+ if remaining >= 9:
90
+ a = (a + int.from_bytes(tail[0:4], "little")) & 0xFFFFFFFF
91
+ b = (b + int.from_bytes(tail[4:8], "little")) & 0xFFFFFFFF
92
+ c_val = int.from_bytes(tail[8:remaining], "little")
93
+ c = (c + c_val) & 0xFFFFFFFF
94
+ elif remaining >= 5:
95
+ a = (a + int.from_bytes(tail[0:4], "little")) & 0xFFFFFFFF
96
+ b_val = int.from_bytes(tail[4:remaining], "little")
97
+ b = (b + b_val) & 0xFFFFFFFF
98
+ else:
99
+ a_val = int.from_bytes(tail[:remaining], "little")
100
+ a = (a + a_val) & 0xFFFFFFFF
101
+
102
+ return _final(a, b, c)
103
+
104
+
105
+ def hash22(call: str) -> int:
106
+ """22-bit hash of ``call`` — used by FT8/FT4 compound-callsign packets."""
107
+ return nhash(call) & MASK22
108
+
109
+
110
+ def hash15(call: str) -> int:
111
+ """15-bit hash of ``call`` — used by WSPR Type 3 packets."""
112
+ return nhash(call) & MASK15
113
+
114
+
115
+ def hash12(call: str) -> int:
116
+ """12-bit hash variant — used by some ARRL-contest message types."""
117
+ return nhash(call) & MASK12
118
+
119
+
120
+ def hash10(call: str) -> int:
121
+ """10-bit hash variant — narrowest WSJT-X hash."""
122
+ return nhash(call) & MASK10
callhash/parse.py ADDED
@@ -0,0 +1,132 @@
1
+ """Shared WSJT-X message-field parser.
2
+
3
+ FT8 / FT4 / MSK144 decoders (``decode_ft8``, ``jt9``) emit a freeform
4
+ message body whose grammar is identical across the modes. Every
5
+ recorder consumer — psk-recorder, meteor-scatter, wspr-recorder — needs
6
+ the same best-effort extraction of tx/rx call, grid, and report, *and*
7
+ the same compound-callsign hash substitution via
8
+ :class:`~callhash.table.CallHashTable`.
9
+
10
+ Before this module the extraction was copy-pasted byte-for-byte into
11
+ each recorder's ``ch_tailer.py`` and the table was built but never read,
12
+ so hashed spots lost their call. Centralising it here means one
13
+ implementation that resolves hashes the same way everywhere — the whole
14
+ point of a shared callhash library.
15
+
16
+ ``parse_message`` is intentionally lossy: the raw (hash-resolved)
17
+ message string is always returned in ``message`` so the caller can
18
+ preserve it verbatim; the parsed call/grid/report fields are best-effort.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import re
23
+ from typing import Any, Dict, Optional
24
+
25
+ # Standard callsigns + WSJT-X compound forms:
26
+ # * standard ITU call: K1ABC, AC0G, JA1AAA
27
+ # * suffix-form compound: K1ABC/QRP, K1ABC/MM
28
+ # * prefix-form compound (region/portable): VE3/K1ABC, G/K1ABC, KH6/AC0G
29
+ _CALL_RE = re.compile(
30
+ r"^"
31
+ r"(?:[A-Z0-9]{1,3}/)?" # optional prefix (e.g. "VE3/", "G/")
32
+ r"[A-Z0-9]{1,3}[0-9][A-Z0-9]{0,4}" # standard call body (XX[X][D][YY[Y][Y]])
33
+ r"(?:/[A-Z0-9]{1,4})?" # optional suffix (e.g. "/QRP", "/MM")
34
+ r"$"
35
+ )
36
+ # Maidenhead 6-char form has uppercase field+square but lowercase
37
+ # subsquare (per IARU convention). Tolerate either case for robustness.
38
+ _GRID_RE = re.compile(r"^[A-R]{2}[0-9]{2}(?:[A-Xa-x]{2})?$")
39
+ _REPORT_RE = re.compile(r"^R?([+-]?\d+)$")
40
+
41
+
42
+ def _is_grid(token: str) -> bool:
43
+ """A Maidenhead locator, but never the sign-off ``RR73``. RR73 has a
44
+ locator's shape (a square in the Arctic Ocean), and WSJT-X never
45
+ sends that square: a grid read from it would place every station
46
+ that signs off near the pole."""
47
+ return bool(_GRID_RE.match(token)) and token.upper() != "RR73"
48
+
49
+
50
+ def _plain(token: str) -> Optional[str]:
51
+ """Return a bare callsign token, or ``None`` if it's still bracketed.
52
+
53
+ ``parse_message`` resolves the whole message first, so any token that
54
+ *still* carries angle brackets here (``<...>`` or a numeric hash we've
55
+ never seen announced) is genuinely unrecoverable — drop it from the
56
+ call fields rather than emitting a placeholder.
57
+ """
58
+ if not token or token.startswith("<"):
59
+ return None
60
+ return token
61
+
62
+
63
+ def parse_message(message: str, table: Optional[Any] = None) -> Dict[str, Any]:
64
+ """Parse a decoded FT8/FT4/MSK144 message body.
65
+
66
+ When ``table`` (a :class:`CallHashTable`) is supplied, the message is
67
+ first run through :meth:`CallHashTable.resolve_message` so bracketed
68
+ hashes that we've learned are substituted with plaintext calls — this
69
+ is what stops us shipping ``<...>`` spots whose call we actually know.
70
+
71
+ Recognized shapes (all approximate; freeform messages return empties):
72
+ "CQ <tx_call> [<grid>]" — directed CQ
73
+ "<rx_call> <tx_call> <grid>" — first contact w/ grid
74
+ "<rx_call> <tx_call> R <grid>" — reply that acknowledges, w/ grid
75
+ "<rx_call> <tx_call> [R]<report>" — signal report
76
+ "<rx_call> <tx_call> [73|RR73]" — close
77
+
78
+ Returns a dict with keys ``message`` (the hash-resolved text — callers
79
+ should store THIS, not the raw input), ``tx_call``, ``rx_call``,
80
+ ``grid`` (all ``""`` when absent) and ``report`` (``None`` when absent).
81
+ """
82
+ if table is not None:
83
+ message = table.resolve_message(message)
84
+
85
+ out: Dict[str, Any] = {
86
+ "message": message,
87
+ "tx_call": "",
88
+ "rx_call": "",
89
+ "grid": "",
90
+ "report": None,
91
+ }
92
+ tokens = message.split()
93
+ if not tokens:
94
+ return out
95
+
96
+ if tokens[0] == "CQ":
97
+ # "CQ [target] <tx_call> [grid]" — `target` may be a region tag
98
+ # like "DX", "EU", "POTA" that isn't a callsign. Scan past
99
+ # non-call tokens until the first call-shaped one (the sender),
100
+ # then look for a grid in the remaining tokens.
101
+ for i, tok in enumerate(tokens[1:], start=1):
102
+ candidate = _plain(tok)
103
+ if candidate is not None and _CALL_RE.match(candidate):
104
+ out["tx_call"] = candidate
105
+ for later in tokens[i + 1:]:
106
+ if _is_grid(later):
107
+ out["grid"] = later
108
+ break
109
+ break
110
+ else:
111
+ # <rx_call> <tx_call> [grid|R grid|report|RR73|73]
112
+ rx_candidate = _plain(tokens[0])
113
+ if rx_candidate is not None and _CALL_RE.match(rx_candidate):
114
+ out["rx_call"] = rx_candidate
115
+ if len(tokens) >= 2:
116
+ tx_candidate = _plain(tokens[1])
117
+ if tx_candidate is not None and _CALL_RE.match(tx_candidate):
118
+ out["tx_call"] = tx_candidate
119
+ if len(tokens) >= 3:
120
+ tail = tokens[2]
121
+ if tail == "R" and len(tokens) >= 4 and _is_grid(tokens[3]):
122
+ out["grid"] = tokens[3]
123
+ elif _is_grid(tail):
124
+ out["grid"] = tail
125
+ else:
126
+ m = _REPORT_RE.match(tail)
127
+ if m:
128
+ try:
129
+ out["report"] = int(m.group(1))
130
+ except ValueError:
131
+ pass
132
+ return out
callhash/table.py ADDED
@@ -0,0 +1,465 @@
1
+ """``CallHashTable`` — accumulator that maps WSJT-X callsign hashes
2
+ back to plaintext from ``<call>`` announcement markers.
3
+
4
+ Two consumers (psk-recorder, wsprdaemon-client) and the same
5
+ mechanism: when a compound callsign is first transmitted, both ends
6
+ broadcast it in plaintext between angle brackets — ``<K1ABC/QRP>``;
7
+ subsequent packets use a 22-bit (FT8/FT4) or 15-bit (WSPR) hash, and
8
+ each side resolves the hash via its in-memory table. Per-invocation
9
+ decoders (jt9, wsprd) start with an empty table, so most hashed
10
+ packets surface as the literal placeholder ``<...>``.
11
+
12
+ This class reconstructs the table at the consumer side by watching
13
+ decoded text for ``<call>`` markers — the announcement is itself a
14
+ plaintext sighting we can hash and store. Persisting the table to
15
+ JSON lets the resolution survive daemon restarts so the cumulative
16
+ mapping grows over time.
17
+
18
+ The class deliberately stays small and stateless beyond the in-memory
19
+ maps so it can be embedded in either client without dragging in an
20
+ import graph.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ import json
25
+ import logging
26
+ import re
27
+ import threading
28
+ from datetime import datetime, timezone
29
+ from pathlib import Path
30
+ from typing import Callable, Dict, Iterable, Optional, Tuple
31
+
32
+ from ._nhash import hash12, hash15, hash22
33
+
34
+ log = logging.getLogger(__name__)
35
+
36
+
37
+ # Plaintext compound-callsign markers in decoded text:
38
+ # - <K1ABC/QRP> announcement (canonical case — two- or three-segment
39
+ # call separated by '/').
40
+ # - <K1ABC> a hashed call already resolved by the decoder
41
+ # from its session table; reading us the call back.
42
+ # - <...> literal "unresolved hash" placeholder (no useful
43
+ # content).
44
+ # The regex below matches the first two but not the literal ellipsis
45
+ # placeholder.
46
+ _BRACKET_CALL_RE = re.compile(r"<([A-Z0-9][A-Z0-9/]{1,15})>")
47
+ _LITERAL_UNRESOLVED = "<...>"
48
+ # A bracketed all-numeric token is a hash the decoder couldn't resolve
49
+ # from its own session table — jt9 ``-Y`` (FST4W / MSK144) and the
50
+ # patched ``decode_ft8`` (FT8 / FT4) both emit the 22-bit hash this way
51
+ # (``<NNNNNNN>``, up to 7 digits since MAX22 = 4194304). No decoder we
52
+ # drive emits the narrower 12-bit form as a number, so a numeric bracket
53
+ # token is always a 22-bit hash.
54
+ _BRACKET_HASH_RE = re.compile(r"<(\d{1,7})>")
55
+
56
+
57
+ class CallHashTable:
58
+ """Per-(client, radiod) cache mapping WSJT-X hashes → plaintext call.
59
+
60
+ Thread-safe in the only way that matters here: ``observe`` and
61
+ ``by_hash*`` may be called concurrently by a tailer thread + a
62
+ parser thread without external locking.
63
+ """
64
+
65
+ SCHEMA_VERSION = 1 # bump when the on-disk JSON shape changes
66
+
67
+ def __init__(self, source_path: Optional[Path] = None) -> None:
68
+ self._source_path: Optional[Path] = (
69
+ Path(source_path) if source_path is not None else None
70
+ )
71
+ self._lock = threading.Lock()
72
+ # Two parallel maps (one per protocol). Same plaintext call
73
+ # is in both — the cost of double-storage is trivial for the
74
+ # observed table sizes (10²-10³ entries) and avoids per-lookup
75
+ # bit-mask choices on the hot path.
76
+ self._by_h22: Dict[int, str] = {}
77
+ self._by_h15: Dict[int, str] = {}
78
+ # 12-bit map kept for API symmetry with the other widths. No
79
+ # decoder we drive emits a 12-bit hash as a resolvable number
80
+ # (decode_ft8's 12-bit site stays ``<...>``), so this is rarely
81
+ # hit in practice, but populating it is free.
82
+ self._by_h12: Dict[int, str] = {}
83
+ # Collision guard. A hash slot claimed by 2+ DISTINCT calls is
84
+ # ambiguous — a reverse lookup can't tell which call the hash
85
+ # meant, so we refuse to resolve it rather than risk emitting the
86
+ # wrong callsign into a spot (a wrong call pollutes wsprnet /
87
+ # pskreporter; a missing call is benign). This matters because
88
+ # OUR table is persistent and accumulates 10²–10³+ calls, so it
89
+ # sees far more collisions than WSJT-X's short-lived per-session
90
+ # table — especially in the 15-bit space (32 768 slots). These
91
+ # sets are recomputed on load, not persisted.
92
+ self._collided_h22: set[int] = set()
93
+ self._collided_h15: set[int] = set()
94
+ self._collided_h12: set[int] = set()
95
+ # First-seen timestamp per call — useful for operator forensics
96
+ # ("when did this compound call first appear?") and bounded
97
+ # eviction in future versions.
98
+ self._first_seen: Dict[str, str] = {}
99
+ self._observations = 0 # total announcements seen (for stats)
100
+ self._dirty = False # save() can short-circuit when no change
101
+
102
+ # ----- observation -----
103
+
104
+ def observe(self, text: str) -> int:
105
+ """Scan ``text`` for ``<call>`` markers and add each to the table.
106
+
107
+ Returns the number of NEW entries inserted by this call.
108
+ Multiple calls in one line are all extracted; existing entries
109
+ are left unchanged (announcements are stable per call).
110
+ """
111
+ added = 0
112
+ with self._lock:
113
+ for match in _BRACKET_CALL_RE.findall(text):
114
+ call = match.strip()
115
+ if not call:
116
+ continue
117
+ if not _looks_like_callsign(call):
118
+ continue
119
+ self._observations += 1
120
+ if self._add_locked(call):
121
+ added += 1
122
+ if added:
123
+ log.debug("callhash: +%d new call(s); table now %d (h22) / %d (h15)",
124
+ added, len(self._by_h22), len(self._by_h15))
125
+ return added
126
+
127
+ def add(self, call: str) -> bool:
128
+ """Direct-add a single plaintext callsign. Returns True if new."""
129
+ with self._lock:
130
+ return self._add_locked(call)
131
+
132
+ def ingest_calls(self, calls: Iterable[str]) -> int:
133
+ """Add a batch of plaintext callsigns. Returns the count of NEW
134
+ entries.
135
+
136
+ Convenience for consumers that discover full calls in decoded
137
+ output (rather than ``<call>`` brackets) — e.g. wspr-recorder's
138
+ per-cycle spot ingest. Non-callsign-shaped entries are skipped.
139
+ """
140
+ added = 0
141
+ with self._lock:
142
+ for call in calls:
143
+ if not call:
144
+ continue
145
+ call = call.strip()
146
+ if not _looks_like_callsign(call):
147
+ continue
148
+ if self._add_locked(call):
149
+ added += 1
150
+ return added
151
+
152
+ def _add_locked(self, call: str) -> bool:
153
+ is_new = call not in self._first_seen
154
+ self._index_call(call)
155
+ if is_new:
156
+ self._first_seen[call] = datetime.now(tz=timezone.utc).isoformat()
157
+ self._dirty = True
158
+ return is_new
159
+
160
+ def _index_call(self, call: str) -> None:
161
+ """Insert ``call`` into the three width-keyed maps, flagging any
162
+ slot already owned by a *different* call as collided.
163
+
164
+ The first call to claim a slot keeps it; a second, distinct call
165
+ that hashes to the same slot makes it ambiguous (added to the
166
+ collided set) so neither resolves. Detection is idempotent and
167
+ order-independent, so live observation and on-disk replay
168
+ produce the same ambiguous-slot set.
169
+ """
170
+ for hfn, fwd, collided in (
171
+ (hash22, self._by_h22, self._collided_h22),
172
+ (hash15, self._by_h15, self._collided_h15),
173
+ (hash12, self._by_h12, self._collided_h12),
174
+ ):
175
+ h = hfn(call)
176
+ existing = fwd.get(h)
177
+ if existing is None:
178
+ fwd[h] = call
179
+ elif existing != call:
180
+ collided.add(h)
181
+
182
+ # ----- lookup -----
183
+
184
+ def by_hash22(self, h: int) -> Optional[str]:
185
+ """Resolve a 22-bit hash (FT8/FT4) to plaintext, or None.
186
+
187
+ Returns None for an ambiguous (collided) slot — see the collision
188
+ guard in ``__init__`` — so we never return a guessed call.
189
+ """
190
+ with self._lock:
191
+ h &= 0x3FFFFF
192
+ if h in self._collided_h22:
193
+ return None
194
+ return self._by_h22.get(h)
195
+
196
+ def by_hash15(self, h: int) -> Optional[str]:
197
+ """Resolve a 15-bit hash (WSPR Type 3) to plaintext, or None.
198
+
199
+ Returns None for an ambiguous (collided) slot.
200
+ """
201
+ with self._lock:
202
+ h &= 0x7FFF
203
+ if h in self._collided_h15:
204
+ return None
205
+ return self._by_h15.get(h)
206
+
207
+ def by_hash12(self, h: int) -> Optional[str]:
208
+ """Resolve a 12-bit hash to plaintext, or None.
209
+
210
+ Returns None for an ambiguous (collided) slot.
211
+ """
212
+ with self._lock:
213
+ h &= 0xFFF
214
+ if h in self._collided_h12:
215
+ return None
216
+ return self._by_h12.get(h)
217
+
218
+ # ----- token / message resolution -----
219
+
220
+ def resolve_token(self, token: str) -> Optional[str]:
221
+ """Resolve a single decoded-message token to a plaintext call.
222
+
223
+ This is the canonical, shared substitution primitive used by
224
+ every consumer (psk / meteor-scatter / wspr) so they behave
225
+ identically. Cases:
226
+
227
+ * ``<NNNNNNN>`` (numeric hash) → :meth:`by_hash22` lookup;
228
+ returns the plaintext call if we've seen it announced, else
229
+ ``None`` (still genuinely unknown — keep the placeholder).
230
+ * ``<...>`` → ``None`` (decoder discarded the hash number;
231
+ nothing to look up).
232
+ * ``<CALL>`` / ``<CALL/QRP>`` → the decoder resolved it from its
233
+ own session table; strip the brackets, **seed our table** with
234
+ the sighting, and return the bare call.
235
+ * bare token (``K1ABC``, ``CQ``, ``EM38``) → returned unchanged.
236
+ * anything else in brackets (garbage) → ``None``.
237
+ """
238
+ if not token:
239
+ return token
240
+ t = token.strip()
241
+ if not (t.startswith("<") and t.endswith(">") and len(t) > 2):
242
+ return t # bare token
243
+ if t == _LITERAL_UNRESOLVED:
244
+ return None
245
+ m = _BRACKET_HASH_RE.fullmatch(t)
246
+ if m:
247
+ return self.by_hash22(int(m.group(1)))
248
+ m = _BRACKET_CALL_RE.fullmatch(t)
249
+ if m:
250
+ call = m.group(1)
251
+ if _looks_like_callsign(call):
252
+ self.add(call) # self-seeding sighting
253
+ return call
254
+ return None # bracketed garbage
255
+
256
+ def resolve_message(self, message: str) -> str:
257
+ """Return ``message`` with every resolvable hash token replaced by
258
+ its plaintext call.
259
+
260
+ Unresolvable tokens (``<...>`` or a numeric hash we've never seen
261
+ announced) are left exactly as-is, so the raw text is preserved
262
+ when we genuinely can't recover the call. Token spacing is
263
+ normalised to single spaces.
264
+ """
265
+ if not message:
266
+ return message
267
+ out = []
268
+ for tok in message.split():
269
+ resolved = self.resolve_token(tok)
270
+ out.append(resolved if resolved is not None else tok)
271
+ return " ".join(out)
272
+
273
+ def __contains__(self, call: str) -> bool:
274
+ with self._lock:
275
+ return call in self._first_seen
276
+
277
+ def __len__(self) -> int:
278
+ with self._lock:
279
+ return len(self._first_seen)
280
+
281
+ @property
282
+ def observations(self) -> int:
283
+ """Total ``<call>`` markers observed (incl. duplicates)."""
284
+ with self._lock:
285
+ return self._observations
286
+
287
+ # ----- normalisation helper -----
288
+
289
+ @staticmethod
290
+ def normalise_brackets(token: str) -> Optional[str]:
291
+ """Canonicalise a possibly-bracketed callsign token.
292
+
293
+ Per the wsprd / jt9 output convention:
294
+ * ``<K1ABC>`` or ``<K1ABC/QRP>`` → strip brackets, return
295
+ the inner call (it has been resolved by the decoder).
296
+ * ``<...>`` → return ``None`` (unresolved hash; no useful
297
+ information at this point — caller may try
298
+ :meth:`by_hash22` / :meth:`by_hash15` if it has a hash to
299
+ look up).
300
+ * Anything else → return as-is.
301
+ """
302
+ if not token:
303
+ return token
304
+ t = token.strip()
305
+ if t == _LITERAL_UNRESOLVED:
306
+ return None
307
+ m = _BRACKET_CALL_RE.fullmatch(t)
308
+ if m:
309
+ return m.group(1)
310
+ return t
311
+
312
+ # ----- persistence -----
313
+
314
+ @classmethod
315
+ def load_or_new(cls, source_path: Path | str) -> "CallHashTable":
316
+ """Load from JSON if it exists, else return a fresh table.
317
+
318
+ The file may be absent, empty, or contain malformed JSON
319
+ (cleared by an operator, half-written by a previous crashed
320
+ save) — in any of those cases we start fresh and log a
321
+ warning. We never delete the operator's file on load failure.
322
+ """
323
+ path = Path(source_path)
324
+ instance = cls(source_path=path)
325
+ if not path.exists():
326
+ return instance
327
+ try:
328
+ data = json.loads(path.read_text())
329
+ except (OSError, json.JSONDecodeError) as e:
330
+ log.warning("callhash: ignoring unreadable %s (%s); starting fresh",
331
+ path, e)
332
+ return instance
333
+ if not isinstance(data, dict):
334
+ return instance
335
+ if data.get("schema_version") != cls.SCHEMA_VERSION:
336
+ log.warning("callhash: %s has schema_version=%r, expected %d; "
337
+ "starting fresh", path,
338
+ data.get("schema_version"), cls.SCHEMA_VERSION)
339
+ return instance
340
+ for call, first_seen in (data.get("calls") or {}).items():
341
+ if not _looks_like_callsign(call):
342
+ continue
343
+ instance._first_seen[call] = first_seen
344
+ instance._index_call(call)
345
+ instance._observations = int(data.get("observations", 0))
346
+ instance._dirty = False
347
+ log.debug("callhash: loaded %d calls from %s", len(instance), path)
348
+ return instance
349
+
350
+ def save(self, path: Optional[Path | str] = None) -> None:
351
+ """Atomically persist the table to disk.
352
+
353
+ Writes to ``<path>.tmp`` then ``os.replace`` so a partial
354
+ write can't corrupt an existing good file. No-op when no
355
+ observations have changed since the last save (or load).
356
+ """
357
+ target = Path(path) if path is not None else self._source_path
358
+ if target is None:
359
+ raise ValueError(
360
+ "save(): no path supplied and no source_path set on table"
361
+ )
362
+ with self._lock:
363
+ if not self._dirty:
364
+ return
365
+ payload = {
366
+ "schema_version": self.SCHEMA_VERSION,
367
+ "saved_at": datetime.now(tz=timezone.utc).isoformat(),
368
+ "observations": self._observations,
369
+ "calls": dict(self._first_seen),
370
+ }
371
+ target.parent.mkdir(parents=True, exist_ok=True)
372
+ tmp = target.with_suffix(target.suffix + ".tmp")
373
+ tmp.write_text(json.dumps(payload, indent=2, sort_keys=True))
374
+ tmp.replace(target)
375
+ with self._lock:
376
+ self._dirty = False
377
+
378
+ # ----- decoder-seed export -----
379
+
380
+ def write_wsprd_hashtable(
381
+ self,
382
+ path: Path | str,
383
+ *,
384
+ exclude: Optional[Callable[[str], bool]] = None,
385
+ ) -> int:
386
+ """Write wsprd's ``hashtable.txt`` (15-bit index → call).
387
+
388
+ Pre-populating this file lets ``wsprd`` resolve Type-3 hashes it
389
+ would otherwise emit as ``<...>`` (wsprd discards the number, so
390
+ the only way to recover those calls is to seed the decoder ahead
391
+ of the run). Format matches wsprd: ``"%5d %s\\n"``.
392
+
393
+ ``exclude`` is an optional predicate — calls for which it returns
394
+ True are omitted. wsprd-recorder passes its wsprnet
395
+ negative-cache filter here so consistently-rejected compounds
396
+ stop being re-emitted; the library itself stays oblivious to that
397
+ policy. Returns the number of entries written.
398
+ """
399
+ path = Path(path)
400
+ with self._lock:
401
+ calls = list(self._first_seen)
402
+ count = 0
403
+ with open(path, "w") as f:
404
+ for call in calls:
405
+ if exclude is not None and exclude(call):
406
+ continue
407
+ f.write(f"{hash15(call):5d} {call}\n")
408
+ count += 1
409
+ return count
410
+
411
+ def write_jt9_calls(
412
+ self,
413
+ path: Path | str,
414
+ grids: Optional[Dict[str, str]] = None,
415
+ *,
416
+ exclude: Optional[Callable[[str], bool]] = None,
417
+ ) -> int:
418
+ """Write jt9's ``fst4w_calls.txt`` (call + grid per line).
419
+
420
+ Same role as :meth:`write_wsprd_hashtable` for the jt9 / FST4W
421
+ side. The library stores no grids, so the caller may pass a
422
+ ``{call: grid}`` map; missing grids are rendered as four spaces
423
+ (jt9's blank-grid convention). ``exclude`` works as above.
424
+ Returns the number of entries written.
425
+ """
426
+ path = Path(path)
427
+ grids = grids or {}
428
+ with self._lock:
429
+ calls = list(self._first_seen)
430
+ count = 0
431
+ with open(path, "w") as f:
432
+ for call in calls:
433
+ if exclude is not None and exclude(call):
434
+ continue
435
+ grid = grids.get(call) or " "
436
+ f.write(f"{call} {grid}\n")
437
+ count += 1
438
+ return count
439
+
440
+ # ----- introspection -----
441
+
442
+ def stats(self) -> Tuple[int, int, int]:
443
+ """Return ``(unique_calls, observations, h22_entries)`` snapshot."""
444
+ with self._lock:
445
+ return len(self._first_seen), self._observations, len(self._by_h22)
446
+
447
+ def calls(self) -> list[str]:
448
+ """Return a snapshot list of known plaintext calls."""
449
+ with self._lock:
450
+ return sorted(self._first_seen)
451
+
452
+
453
+ # ── helpers ────────────────────────────────────────────────────────────────
454
+
455
+ # Loose callsign sanity check — both standard ITU calls and compound
456
+ # variants we care about. We let through anything that's mostly
457
+ # alphanumerics with at most a few '/' separators; the decoder's own
458
+ # bracket markers already filter out garbage upstream.
459
+ _CALLSIGN_VALIDATOR = re.compile(
460
+ r"^[A-Z0-9](?:[A-Z0-9/]{0,15})$"
461
+ )
462
+
463
+
464
+ def _looks_like_callsign(call: str) -> bool:
465
+ return bool(_CALLSIGN_VALIDATOR.fullmatch(call))
@@ -0,0 +1,134 @@
1
+ Metadata-Version: 2.4
2
+ Name: callhash
3
+ Version: 1.1.0
4
+ Summary: WSJT-X callsign-hash mechanism (Bob Jenkins lookup3, seed 146) plus a per-station accumulator that resolves hashes from <call> announcement markers in decoded jt9 / wsprd output.
5
+ Author: Rob Robinett AI6VN
6
+ Author-email: Michael Hauan AC0G <ac0g@hauan.org>
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://github.com/HamSCI/callhash
9
+ Project-URL: Bug Tracker, https://github.com/HamSCI/callhash/issues
10
+ Keywords: wsjt-x,ham radio,ft8,ft4,wspr,callsign,hash
11
+ Requires-Python: >=3.10
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Provides-Extra: dev
15
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
16
+ Dynamic: license-file
17
+
18
+ # callhash
19
+
20
+ Pure-Python WSJT-X compound-callsign hash resolution.
21
+
22
+ WSJT-X's FT4/FT8/WSPR packet formats compress compound callsigns
23
+ (e.g. `K1ABC/QRP`, `VE3/K1ABC`) into 22-bit (FT8/FT4) or 15-bit
24
+ (WSPR Type 3) hashes. Receivers maintain a session table that maps
25
+ the hash back to plaintext when the call has been observed in a
26
+ `<call>` first-occurrence message. Both `jt9` and `wsprd` use the
27
+ same hash function — Bob Jenkins lookup3, seed 146.
28
+
29
+ When a per-slot decoder invocation starts with an empty session
30
+ table (the typical case for `psk-recorder`, `meteor-scatter`,
31
+ `wspr-recorder`, or any consumer that drives the decoder one cycle
32
+ at a time), most hashed packets surface as the literal `<...>`
33
+ placeholder — or, when the decoder is told to emit the number
34
+ (`jt9 -Y`, the patched `decode_ft8`), as `<NNNNNNN>`. This library
35
+ reconstructs the table on the consumer side from the same
36
+ announcement markers WSJT-X uses, and substitutes the numeric
37
+ hashes back to plaintext.
38
+
39
+ All three HamSCI recorders use this one library for build
40
+ (`observe`), lookup (`by_hashNN`), and substitution
41
+ (`resolve_token` / `resolve_message` / `parse_message`) — the same
42
+ mechanism everywhere, so a compound call learned on one mode/band
43
+ resolves hashes on another. Every mode hashes with the identical
44
+ `nhash`; only the width differs (22-bit FT8/FT4/MSK144/FST4W,
45
+ 15-bit WSPR-2), and the table keys each width separately.
46
+
47
+ ## Install
48
+
49
+ ```
50
+ pip install callhash
51
+ ```
52
+
53
+ Or from a sibling checkout (HamSCI deployment pattern):
54
+
55
+ ```
56
+ pip install -e /opt/git/sigmond/callhash
57
+ ```
58
+
59
+ ## Usage
60
+
61
+ ```python
62
+ from callhash import CallHashTable, hash22, hash15, nhash
63
+
64
+ # Persistent per-station cache.
65
+ table = CallHashTable.load_or_new("/var/lib/myclient/callhash.json")
66
+
67
+ # Feed any text containing <call> markers; the table extracts them.
68
+ table.observe("260507 1234 -12 +0.45 1250 ` <K1ABC/QRP> CQ FT8")
69
+
70
+ # Look a hash up if you happen to have one.
71
+ table.by_hash22(hash22("K1ABC/QRP")) # → "K1ABC/QRP"
72
+ table.by_hash15(hash15("K1ABC/QRP")) # → "K1ABC/QRP"
73
+
74
+ # Resolve a decoded message: substitutes <NNNNNNN>/<CALL>, drops <...>.
75
+ table.resolve_message("AC0G <2288505> R-12") # → "AC0G K9AN R-12" (if learned)
76
+
77
+ # Or parse a whole message into call fields, resolving hashes via the table.
78
+ from callhash import parse_message
79
+ parse_message("AC0G <2288505> R-12", table=table)
80
+ # → {"message": "AC0G K9AN R-12", "rx_call": "AC0G", "tx_call": "K9AN",
81
+ # "grid": "", "report": -12}
82
+
83
+ # Persist for the next invocation.
84
+ table.save()
85
+ ```
86
+
87
+ ## Public API
88
+
89
+ | Symbol | Purpose |
90
+ | --------------------- | ---------------------------------------------------------------------- |
91
+ | `nhash(key, initval)` | Bob Jenkins lookup3 32-bit hash. `initval=146` matches WSJT-X. |
92
+ | `hash22(call)` | Convenience: `nhash(call) & 0x3FFFFF` (FT8/FT4 compound-call width). |
93
+ | `hash15(call)` | Convenience: `nhash(call) & 0x7FFF` (WSPR Type 3 width). |
94
+ | `hash12(call)` | Convenience: 12-bit mask (some ARRL contest variants). |
95
+ | `hash10(call)` | Convenience: 10-bit mask (narrowest WSJT-X variant). |
96
+ | `CallHashTable` | Persistent accumulator + lookup + substitution + collision guard. |
97
+ | `parse_message(msg, table=None)` | Shared WSJT-X message-field parser (tx/rx/grid/report) + hash substitution. |
98
+
99
+ `CallHashTable` resolves with `resolve_token` / `resolve_message`,
100
+ seeds decoders with `write_wsprd_hashtable` / `write_jt9_calls`
101
+ (both take an `exclude=` predicate), and **guards collisions**: a
102
+ hash slot claimed by two distinct calls is ambiguous, so
103
+ `by_hashNN` returns `None` rather than a guessed (wrong) call. This
104
+ matters because a persistent table accumulates far more calls than
105
+ WSJT-X's per-session table, so collisions — especially in the
106
+ 15-bit space — are common.
107
+
108
+ ## Correctness
109
+
110
+ `nhash` is bit-exact against WSJT-X's canonical
111
+ `lib/wsprcode/nhash.c` (the unmasked variant — note that
112
+ `lib/wsprd/nhash.c` has a 15-bit mask baked into its return). 23
113
+ reference vectors verified, including 11/12/13-byte boundary cases
114
+ that exercise distinct final-block branches in the C code.
115
+
116
+ The `CallHashTable` covers `<call>` announcement extraction,
117
+ hash lookups, token/message substitution, the collision guard,
118
+ atomic JSON persistence (write-tempfile + rename), corrupt-JSON /
119
+ schema-mismatch recovery, and concurrent-observe / concurrent-lookup
120
+ safety. 89 tests total; no live ClickHouse / WSJT-X server required
121
+ to run them.
122
+
123
+ ## Why this lives in its own repo
124
+
125
+ The hash function and bracket-resolution logic are WSJT-X's, not
126
+ sigmond's. They're useful for any FT8/FT4/WSPR consumer — the
127
+ sigmond client suite is the primary user today, but a future
128
+ `hs-uploader` library or any independent log analyser will
129
+ benefit equally. Pure stdlib, no runtime deps, ~200 lines of
130
+ code; small enough to live cleanly on its own.
131
+
132
+ ## License
133
+
134
+ MIT.
@@ -0,0 +1,9 @@
1
+ callhash/__init__.py,sha256=vcJ-jBhpE3D4ynmPlXb8JNfJnQRumyDmlvrz1vKiKRY,1959
2
+ callhash/_nhash.py,sha256=lqSYeByvDgb6ZkSIzHmwQHxwhxcRvYUe5MMpZix_vmY,4841
3
+ callhash/parse.py,sha256=yUHnifLF-XxnvCbdd9vxIrEkCQEGNB6PfxmQP6bx5_E,5573
4
+ callhash/table.py,sha256=4v79wU7etRzhSMn4uYnh2yI5pQGnFc7ZvNM35iTl-rE,18731
5
+ callhash-1.1.0.dist-info/licenses/LICENSE,sha256=8rBDzvlFRUZM6znjnevy0o2nzZfY4ChmNGAgTHEcl_c,1105
6
+ callhash-1.1.0.dist-info/METADATA,sha256=Im0FZRCVDR4xNhm7yjhL7fgOzVGmhrybg8OWl5o0mPU,5681
7
+ callhash-1.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ callhash-1.1.0.dist-info/top_level.txt,sha256=xo0toeXrc8UE0olLDsrlz-oo7xAISHra1X405Tn4wWI,9
9
+ callhash-1.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 HamSCI / Michael Hauan AC0G / Rob Robinett AI6VN
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ callhash