pynltp 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- necklink/__init__.py +103 -0
- necklink/__main__.py +4 -0
- necklink/baselines.py +551 -0
- necklink/c_export.py +594 -0
- necklink/cli.py +246 -0
- necklink/codec.py +536 -0
- necklink/connectome_priors.py +143 -0
- necklink/internet.py +514 -0
- necklink/nltp.py +1737 -0
- necklink/quic_adapter.py +547 -0
- necklink/socket_api.py +731 -0
- necklink/workloads.py +228 -0
- nltp/__init__.py +3 -0
- nltp/__main__.py +5 -0
- pynltp/__init__.py +3 -0
- pynltp/__main__.py +5 -0
- pynltp-1.2.0.dist-info/METADATA +392 -0
- pynltp-1.2.0.dist-info/RECORD +22 -0
- pynltp-1.2.0.dist-info/WHEEL +5 -0
- pynltp-1.2.0.dist-info/entry_points.txt +4 -0
- pynltp-1.2.0.dist-info/licenses/LICENSE +21 -0
- pynltp-1.2.0.dist-info/top_level.txt +3 -0
necklink/__init__.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""
|
|
2
|
+
NeckLink: Connectome-Inspired Zero-Retransmission Data Transfer Protocol & Codec.
|
|
3
|
+
|
|
4
|
+
Translates three architectural principles discovered in the adult Drosophila
|
|
5
|
+
brain-to-body neck connective (325,354 neurons across MaleCNS v1.0 & Female BANC v888)
|
|
6
|
+
into a universal ultra-low-bandwidth, packet-loss-healing wire protocol:
|
|
7
|
+
1. Contractive Twin-Mirror Efference Predictor (0-LSB bit-exact integer state sync)
|
|
8
|
+
2. Epistatic Cryptic-Backup Echo & Subspace Projection (cliff-free loss recovery)
|
|
9
|
+
3. Dale's-Law Push-Pull + Disinhibitory Shunting Gain Rescue (0-ACK variance/DC healing)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from .connectome_priors import (
|
|
13
|
+
DROSOPHILA_NECK_PRIORS,
|
|
14
|
+
build_epistatic_cryptic_matrix_q8,
|
|
15
|
+
measure_empirical_connectome_neck_stats,
|
|
16
|
+
)
|
|
17
|
+
from .codec import (
|
|
18
|
+
NeckLinkConfig,
|
|
19
|
+
NeckLinkEncoder,
|
|
20
|
+
NeckLinkDecoder,
|
|
21
|
+
NeckLinkSplitBottleneck,
|
|
22
|
+
c_div,
|
|
23
|
+
)
|
|
24
|
+
from .baselines import (
|
|
25
|
+
RawFloat32UDPCodec,
|
|
26
|
+
Int8ZlibBlockCodec,
|
|
27
|
+
SendOnDeltaUDPCodec,
|
|
28
|
+
ADPCMLeakyUDPCodec,
|
|
29
|
+
ReedSolomonGF256Codec,
|
|
30
|
+
TCPARQAnalyticalEvaluator,
|
|
31
|
+
simulate_packet_drops,
|
|
32
|
+
compute_link_economics,
|
|
33
|
+
)
|
|
34
|
+
from .c_export import export_necklink_c99_header
|
|
35
|
+
from .internet import (
|
|
36
|
+
AEADCipherContext,
|
|
37
|
+
NLTPRelayHub,
|
|
38
|
+
ReplayWindow64,
|
|
39
|
+
STUNDiscoveryResult,
|
|
40
|
+
TokenBucketPacer,
|
|
41
|
+
discover_wan_endpoint,
|
|
42
|
+
generate_nltp_key,
|
|
43
|
+
)
|
|
44
|
+
from .nltp import NLTPClient, NLTPEndpoint, NLTPServer, ReceivedItem, pack_nltp_frame, unpack_nltp_frame
|
|
45
|
+
from .socket_api import AsyncNeckSocket, NeckSocket, NeckUDPServer
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
from .quic_adapter import (
|
|
49
|
+
LossyUDPRelay,
|
|
50
|
+
NeckLinkQuicProtocol,
|
|
51
|
+
NeckLinkQuicServer,
|
|
52
|
+
connect_necklink_quic,
|
|
53
|
+
generate_ephemeral_quic_cert,
|
|
54
|
+
)
|
|
55
|
+
except ImportError: # Optional dependency (`pip install aioquic`)
|
|
56
|
+
LossyUDPRelay = None # type: ignore[assignment,misc]
|
|
57
|
+
NeckLinkQuicProtocol = None # type: ignore[assignment,misc]
|
|
58
|
+
NeckLinkQuicServer = None # type: ignore[assignment,misc]
|
|
59
|
+
connect_necklink_quic = None # type: ignore[assignment]
|
|
60
|
+
generate_ephemeral_quic_cert = None # type: ignore[assignment]
|
|
61
|
+
|
|
62
|
+
__version__ = "1.2.0"
|
|
63
|
+
|
|
64
|
+
__all__ = [
|
|
65
|
+
"DROSOPHILA_NECK_PRIORS",
|
|
66
|
+
"build_epistatic_cryptic_matrix_q8",
|
|
67
|
+
"measure_empirical_connectome_neck_stats",
|
|
68
|
+
"NeckLinkConfig",
|
|
69
|
+
"NeckLinkEncoder",
|
|
70
|
+
"NeckLinkDecoder",
|
|
71
|
+
"NeckLinkSplitBottleneck",
|
|
72
|
+
"c_div",
|
|
73
|
+
"RawFloat32UDPCodec",
|
|
74
|
+
"Int8ZlibBlockCodec",
|
|
75
|
+
"SendOnDeltaUDPCodec",
|
|
76
|
+
"ADPCMLeakyUDPCodec",
|
|
77
|
+
"ReedSolomonGF256Codec",
|
|
78
|
+
"TCPARQAnalyticalEvaluator",
|
|
79
|
+
"simulate_packet_drops",
|
|
80
|
+
"compute_link_economics",
|
|
81
|
+
"export_necklink_c99_header",
|
|
82
|
+
"NeckSocket",
|
|
83
|
+
"NeckUDPServer",
|
|
84
|
+
"AsyncNeckSocket",
|
|
85
|
+
"NLTPServer",
|
|
86
|
+
"NLTPClient",
|
|
87
|
+
"NLTPEndpoint",
|
|
88
|
+
"ReceivedItem",
|
|
89
|
+
"pack_nltp_frame",
|
|
90
|
+
"unpack_nltp_frame",
|
|
91
|
+
"AEADCipherContext",
|
|
92
|
+
"ReplayWindow64",
|
|
93
|
+
"TokenBucketPacer",
|
|
94
|
+
"STUNDiscoveryResult",
|
|
95
|
+
"discover_wan_endpoint",
|
|
96
|
+
"generate_nltp_key",
|
|
97
|
+
"NLTPRelayHub",
|
|
98
|
+
"NeckLinkQuicServer",
|
|
99
|
+
"NeckLinkQuicProtocol",
|
|
100
|
+
"connect_necklink_quic",
|
|
101
|
+
"generate_ephemeral_quic_cert",
|
|
102
|
+
"LossyUDPRelay",
|
|
103
|
+
]
|
necklink/__main__.py
ADDED
necklink/baselines.py
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Industry-Standard Telemetry & Data-Transfer Baselines, Channel Loss Models,
|
|
3
|
+
and Commercial Link Economics Calculators (Iridium SBD, LoRaWAN, Rural Cellular, Split-AI).
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import struct
|
|
8
|
+
import zlib
|
|
9
|
+
from typing import Dict, List, Optional, Tuple
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from .codec import c_div, clip_i32
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------------------------
|
|
17
|
+
# Packet-Loss Channel Simulators (Bernoulli + Gilbert-Elliott Burst Fading)
|
|
18
|
+
# ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
def simulate_packet_drops(
|
|
21
|
+
n_steps: int,
|
|
22
|
+
loss_rate: float,
|
|
23
|
+
mode: str = "random",
|
|
24
|
+
mean_burst_len: float = 8.0,
|
|
25
|
+
seed: int = 42,
|
|
26
|
+
) -> np.ndarray:
|
|
27
|
+
"""
|
|
28
|
+
Returns boolean array `dropped[t]` of length `n_steps` (True = packet lost).
|
|
29
|
+
Supports:
|
|
30
|
+
- mode='random': i.i.d. Bernoulli packet loss
|
|
31
|
+
- mode='gilbert_elliott': 2-state Markov bursty wireless fading channel
|
|
32
|
+
calibrated to target stationary `loss_rate` and `mean_burst_len`.
|
|
33
|
+
"""
|
|
34
|
+
rng = np.random.default_rng(seed)
|
|
35
|
+
if loss_rate <= 0.0:
|
|
36
|
+
return np.zeros(n_steps, dtype=bool)
|
|
37
|
+
if loss_rate >= 1.0:
|
|
38
|
+
return np.ones(n_steps, dtype=bool)
|
|
39
|
+
|
|
40
|
+
if mode == "random":
|
|
41
|
+
return rng.random(n_steps) < loss_rate
|
|
42
|
+
|
|
43
|
+
if mode == "gilbert_elliott":
|
|
44
|
+
# Bad state drops with p_B = 0.92; Good state drops with p_G = 0.01
|
|
45
|
+
p_g, p_b = 0.01, 0.92
|
|
46
|
+
if loss_rate <= p_g:
|
|
47
|
+
return rng.random(n_steps) < loss_rate
|
|
48
|
+
pi_b = min(0.95, max(0.01, (loss_rate - p_g) / (p_b - p_g)))
|
|
49
|
+
# Transition probabilities: r = P(B -> G) = 1 / mean_burst_len
|
|
50
|
+
r_bg = 1.0 / max(1.5, mean_burst_len)
|
|
51
|
+
p_gb = (pi_b * r_bg) / max(1e-6, (1.0 - pi_b))
|
|
52
|
+
p_gb = min(0.95, max(1e-5, p_gb))
|
|
53
|
+
|
|
54
|
+
dropped = np.zeros(n_steps, dtype=bool)
|
|
55
|
+
in_bad = bool(rng.random() < pi_b)
|
|
56
|
+
u_trans = rng.random(n_steps)
|
|
57
|
+
u_drop = rng.random(n_steps)
|
|
58
|
+
for t in range(n_steps):
|
|
59
|
+
dropped[t] = u_drop[t] < (p_b if in_bad else p_g)
|
|
60
|
+
if in_bad:
|
|
61
|
+
if u_trans[t] < r_bg:
|
|
62
|
+
in_bad = False
|
|
63
|
+
else:
|
|
64
|
+
if u_trans[t] < p_gb:
|
|
65
|
+
in_bad = True
|
|
66
|
+
return dropped
|
|
67
|
+
|
|
68
|
+
raise ValueError(f"Unknown channel loss mode: {mode!r}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Baseline 1: Raw IEEE-754 Float32 over UDP (Zero-Order Hold on Loss)
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
class RawFloat32UDPCodec:
|
|
76
|
+
def __init__(self, n_channels: int) -> None:
|
|
77
|
+
self.n_channels = n_channels
|
|
78
|
+
|
|
79
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray) -> Tuple[np.ndarray, int, int]:
|
|
80
|
+
t_steps, n = x_seq.shape
|
|
81
|
+
rx = np.zeros_like(x_seq, dtype=np.float64)
|
|
82
|
+
last = np.zeros(n, dtype=np.float64)
|
|
83
|
+
total_bytes = 0
|
|
84
|
+
tx_pkts = 0
|
|
85
|
+
pkt_size = 2 + 4 * n # 2B seq + 4B*N float32
|
|
86
|
+
for t in range(t_steps):
|
|
87
|
+
pkt = struct.pack(f"<H{n}f", t & 0xFFFF, *x_seq[t].astype(np.float32))
|
|
88
|
+
total_bytes += len(pkt)
|
|
89
|
+
tx_pkts += 1
|
|
90
|
+
if not dropped[t]:
|
|
91
|
+
vals = struct.unpack(f"<H{n}f", pkt)[1:]
|
|
92
|
+
last = np.array(vals, dtype=np.float64)
|
|
93
|
+
rx[t] = last
|
|
94
|
+
assert pkt_size == 2 + 4 * n
|
|
95
|
+
return rx, total_bytes, tx_pkts
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
# Baseline 2: Standardized Int8 + Zlib (DEFLATE) 16-Tick Micro-Blocks over UDP
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
class Int8ZlibBlockCodec:
|
|
103
|
+
def __init__(self, n_channels: int, obs_mean: np.ndarray, obs_std: np.ndarray, block_len: int = 16) -> None:
|
|
104
|
+
self.n_channels = n_channels
|
|
105
|
+
self.mean = np.asarray(obs_mean, dtype=np.float64)
|
|
106
|
+
self.std = np.maximum(np.asarray(obs_std, dtype=np.float64), 1e-6)
|
|
107
|
+
self.block_len = block_len
|
|
108
|
+
|
|
109
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray) -> Tuple[np.ndarray, int, int]:
|
|
110
|
+
t_steps, n = x_seq.shape
|
|
111
|
+
rx = np.zeros_like(x_seq, dtype=np.float64)
|
|
112
|
+
last = self.mean.copy()
|
|
113
|
+
total_bytes = 0
|
|
114
|
+
tx_pkts = 0
|
|
115
|
+
|
|
116
|
+
for start in range(0, t_steps, self.block_len):
|
|
117
|
+
end = min(t_steps, start + self.block_len)
|
|
118
|
+
blk = x_seq[start:end]
|
|
119
|
+
# Quantize +/- 3.2 std into int8 [-127, 127]
|
|
120
|
+
z_norm = (blk - self.mean) / self.std
|
|
121
|
+
q_blk = np.clip(np.round(z_norm * (127.0 / 3.2)), -127, 127).astype(np.int8)
|
|
122
|
+
comp = zlib.compress(q_blk.tobytes(), level=6)
|
|
123
|
+
pkt_bytes = 3 + len(comp) # 2B block_seq + 1B len + compressed payload
|
|
124
|
+
total_bytes += pkt_bytes
|
|
125
|
+
tx_pkts += 1
|
|
126
|
+
|
|
127
|
+
# Packet arrives or drops at end of block
|
|
128
|
+
pkt_idx = end - 1
|
|
129
|
+
if not dropped[pkt_idx]:
|
|
130
|
+
dec_q = np.frombuffer(zlib.decompress(comp), dtype=np.int8).reshape(end - start, n)
|
|
131
|
+
rec_blk = (dec_q.astype(np.float64) * (3.2 / 127.0)) * self.std + self.mean
|
|
132
|
+
rx[start:end] = rec_blk
|
|
133
|
+
last = rec_blk[-1].copy()
|
|
134
|
+
else:
|
|
135
|
+
rx[start:end] = last
|
|
136
|
+
|
|
137
|
+
return rx, total_bytes, tx_pkts
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
# Baseline 3: Standard Neuromorphic / IoT Send-on-Delta (MQTT Sparkplug / AER)
|
|
142
|
+
# ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
class SendOnDeltaUDPCodec:
|
|
145
|
+
"""
|
|
146
|
+
Standard event-triggered Send-on-Delta with identical deadband_q8 and quant_step_q8
|
|
147
|
+
to NeckLink, using a non-leaky integrator (alpha = 1.0) without cryptic backup
|
|
148
|
+
or disinhibitory gain rescue.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def __init__(
|
|
152
|
+
self,
|
|
153
|
+
n_channels: int,
|
|
154
|
+
obs_mean: np.ndarray,
|
|
155
|
+
obs_std: np.ndarray,
|
|
156
|
+
deadband_q8: int = 32,
|
|
157
|
+
quant_step_q8: int = 8,
|
|
158
|
+
) -> None:
|
|
159
|
+
self.n_channels = n_channels
|
|
160
|
+
self.mean_q8 = np.round(np.asarray(obs_mean, dtype=np.float64) * 256.0).astype(np.int32)
|
|
161
|
+
std_safe = np.maximum(np.asarray(obs_std, dtype=np.float64), 1e-6)
|
|
162
|
+
self.inv_std_q8 = np.clip(np.round(256.0 / std_safe), 1, 32767).astype(np.int32)
|
|
163
|
+
self.std_q8 = np.clip(np.round(std_safe * 256.0), 1, 32767).astype(np.int32)
|
|
164
|
+
self.deadband_q8 = deadband_q8
|
|
165
|
+
self.quant_step_q8 = quant_step_q8
|
|
166
|
+
|
|
167
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray) -> Tuple[np.ndarray, int, int]:
|
|
168
|
+
t_steps, n = x_seq.shape
|
|
169
|
+
tx_hat = np.zeros(n, dtype=np.int32)
|
|
170
|
+
rx_hat = np.zeros(n, dtype=np.int32)
|
|
171
|
+
rx_out = np.zeros_like(x_seq, dtype=np.float64)
|
|
172
|
+
total_bytes = 0
|
|
173
|
+
tx_pkts = 0
|
|
174
|
+
half = self.quant_step_q8 // 2
|
|
175
|
+
|
|
176
|
+
for t in range(t_steps):
|
|
177
|
+
x_raw_q8 = np.round(x_seq[t] * 256.0).astype(np.int64)
|
|
178
|
+
events: List[Tuple[int, int]] = []
|
|
179
|
+
for i in range(n):
|
|
180
|
+
zi = clip_i32(c_div(int(x_raw_q8[i] - self.mean_q8[i]) * int(self.inv_std_q8[i]), 256), -16384, 16383)
|
|
181
|
+
ri = zi - int(tx_hat[i])
|
|
182
|
+
if abs(ri) >= self.deadband_q8:
|
|
183
|
+
qi = ((ri + half) // self.quant_step_q8) if ri >= 0 else -(((-ri) + half) // self.quant_step_q8)
|
|
184
|
+
qi = clip_i32(qi, -127, 127)
|
|
185
|
+
if qi != 0:
|
|
186
|
+
events.append((i, qi))
|
|
187
|
+
tx_hat[i] = clip_i32(int(tx_hat[i]) + qi * self.quant_step_q8, -32768, 32767)
|
|
188
|
+
|
|
189
|
+
if events:
|
|
190
|
+
# 2B seq + 1B count + 2B * len(events)
|
|
191
|
+
pkt_len = 3 + 2 * len(events)
|
|
192
|
+
total_bytes += pkt_len
|
|
193
|
+
tx_pkts += 1
|
|
194
|
+
if not dropped[t]:
|
|
195
|
+
for idx, qi in events:
|
|
196
|
+
rx_hat[idx] = clip_i32(int(rx_hat[idx]) + qi * self.quant_step_q8, -32768, 32767)
|
|
197
|
+
|
|
198
|
+
for i in range(n):
|
|
199
|
+
xq = c_div(int(rx_hat[i]) * int(self.std_q8[i]), 256) + int(self.mean_q8[i])
|
|
200
|
+
rx_out[t, i] = float(xq) / 256.0
|
|
201
|
+
|
|
202
|
+
return rx_out, total_bytes, tx_pkts
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
# Baseline 4: Classical Telecom 4-Bit Leaky ADPCM over UDP
|
|
207
|
+
# ---------------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
class ADPCMLeakyUDPCodec:
|
|
210
|
+
"""Fixed-rate 4-bit per channel leaky DPCM (packed 2 channels/byte + 2B seq)."""
|
|
211
|
+
|
|
212
|
+
def __init__(self, n_channels: int, obs_mean: np.ndarray, obs_std: np.ndarray) -> None:
|
|
213
|
+
self.n_channels = n_channels
|
|
214
|
+
self.mean_q8 = np.round(np.asarray(obs_mean, dtype=np.float64) * 256.0).astype(np.int32)
|
|
215
|
+
std_safe = np.maximum(np.asarray(obs_std, dtype=np.float64), 1e-6)
|
|
216
|
+
self.inv_std_q8 = np.clip(np.round(256.0 / std_safe), 1, 32767).astype(np.int32)
|
|
217
|
+
self.std_q8 = np.clip(np.round(std_safe * 256.0), 1, 32767).astype(np.int32)
|
|
218
|
+
self.alpha_q8 = 242
|
|
219
|
+
self.step_q8 = 28
|
|
220
|
+
|
|
221
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray) -> Tuple[np.ndarray, int, int]:
|
|
222
|
+
t_steps, n = x_seq.shape
|
|
223
|
+
tx_m = np.zeros(n, dtype=np.int32)
|
|
224
|
+
rx_m = np.zeros(n, dtype=np.int32)
|
|
225
|
+
rx_out = np.zeros_like(x_seq, dtype=np.float64)
|
|
226
|
+
pkt_bytes = 2 + (n + 1) // 2
|
|
227
|
+
total_bytes = 0
|
|
228
|
+
tx_pkts = 0
|
|
229
|
+
|
|
230
|
+
for t in range(t_steps):
|
|
231
|
+
x_raw_q8 = np.round(x_seq[t] * 256.0).astype(np.int64)
|
|
232
|
+
q4 = np.zeros(n, dtype=np.int32)
|
|
233
|
+
for i in range(n):
|
|
234
|
+
tx_m[i] = c_div(self.alpha_q8 * int(tx_m[i]), 256)
|
|
235
|
+
rx_m[i] = c_div(self.alpha_q8 * int(rx_m[i]), 256)
|
|
236
|
+
zi = clip_i32(c_div(int(x_raw_q8[i] - self.mean_q8[i]) * int(self.inv_std_q8[i]), 256), -4096, 4095)
|
|
237
|
+
ri = zi - int(tx_m[i])
|
|
238
|
+
qi = clip_i32(int(round(ri / self.step_q8)), -7, 7)
|
|
239
|
+
q4[i] = qi
|
|
240
|
+
tx_m[i] = clip_i32(int(tx_m[i]) + qi * self.step_q8, -32768, 32767)
|
|
241
|
+
|
|
242
|
+
total_bytes += pkt_bytes
|
|
243
|
+
tx_pkts += 1
|
|
244
|
+
if not dropped[t]:
|
|
245
|
+
for i in range(n):
|
|
246
|
+
rx_m[i] = clip_i32(int(rx_m[i]) + int(q4[i]) * self.step_q8, -32768, 32767)
|
|
247
|
+
|
|
248
|
+
for i in range(n):
|
|
249
|
+
xq = c_div(int(rx_m[i]) * int(self.std_q8[i]), 256) + int(self.mean_q8[i])
|
|
250
|
+
rx_out[t, i] = float(xq) / 256.0
|
|
251
|
+
|
|
252
|
+
return rx_out, total_bytes, tx_pkts
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
# ---------------------------------------------------------------------------
|
|
256
|
+
# Baseline 5: Exact Systematic GF(2^8) Reed-Solomon Erasure Coding (n=12, k=8)
|
|
257
|
+
# ---------------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
class _GF256:
|
|
260
|
+
"""Exact Galois Field GF(2^8) with primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 (0x11d)."""
|
|
261
|
+
|
|
262
|
+
def __init__(self) -> None:
|
|
263
|
+
self.exp = np.zeros(512, dtype=np.uint8)
|
|
264
|
+
self.log = np.zeros(256, dtype=np.int32)
|
|
265
|
+
x = 1
|
|
266
|
+
for i in range(255):
|
|
267
|
+
self.exp[i] = x
|
|
268
|
+
self.log[x] = i
|
|
269
|
+
x <<= 1
|
|
270
|
+
if x & 0x100:
|
|
271
|
+
x ^= 0x11D
|
|
272
|
+
for i in range(255, 512):
|
|
273
|
+
self.exp[i] = self.exp[i - 255]
|
|
274
|
+
|
|
275
|
+
def mul(self, a: int, b: int) -> int:
|
|
276
|
+
if a == 0 or b == 0:
|
|
277
|
+
return 0
|
|
278
|
+
return int(self.exp[self.log[a] + self.log[b]])
|
|
279
|
+
|
|
280
|
+
def inv(self, a: int) -> int:
|
|
281
|
+
if a == 0:
|
|
282
|
+
raise ZeroDivisionError("GF(256) inverse of 0")
|
|
283
|
+
return int(self.exp[255 - self.log[a]])
|
|
284
|
+
|
|
285
|
+
def matmul(self, a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
|
286
|
+
"""Matrix product over GF(256): (R, K) x (K, C) -> (R, C) uint8."""
|
|
287
|
+
r, k = a.shape
|
|
288
|
+
_, c = b.shape
|
|
289
|
+
out = np.zeros((r, c), dtype=np.uint8)
|
|
290
|
+
for i in range(r):
|
|
291
|
+
for p in range(k):
|
|
292
|
+
ap = int(a[i, p])
|
|
293
|
+
if ap == 0:
|
|
294
|
+
continue
|
|
295
|
+
log_ap = self.log[ap]
|
|
296
|
+
nz = b[p] != 0
|
|
297
|
+
if np.any(nz):
|
|
298
|
+
logs = self.log[b[p, nz]] + log_ap
|
|
299
|
+
out[i, nz] ^= self.exp[logs]
|
|
300
|
+
return out
|
|
301
|
+
|
|
302
|
+
def invert_matrix(self, m: np.ndarray) -> np.ndarray:
|
|
303
|
+
k = m.shape[0]
|
|
304
|
+
aug = np.hstack([m.astype(np.uint8).copy(), np.eye(k, dtype=np.uint8)])
|
|
305
|
+
for col in range(k):
|
|
306
|
+
piv = col
|
|
307
|
+
while piv < k and aug[piv, col] == 0:
|
|
308
|
+
piv += 1
|
|
309
|
+
if piv == k:
|
|
310
|
+
raise np.linalg.LinAlgError("Singular GF(256) submatrix")
|
|
311
|
+
if piv != col:
|
|
312
|
+
aug[[col, piv]] = aug[[piv, col]]
|
|
313
|
+
inv_p = self.inv(int(aug[col, col]))
|
|
314
|
+
for j in range(2 * k):
|
|
315
|
+
aug[col, j] = self.mul(int(aug[col, j]), inv_p)
|
|
316
|
+
for row in range(k):
|
|
317
|
+
if row != col and aug[row, col] != 0:
|
|
318
|
+
factor = int(aug[row, col])
|
|
319
|
+
for j in range(2 * k):
|
|
320
|
+
aug[row, j] ^= self.mul(factor, int(aug[col, j]))
|
|
321
|
+
return aug[:, k:]
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
_GF = _GF256()
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
class ReedSolomonGF256Codec:
|
|
328
|
+
"""
|
|
329
|
+
Systematic Cauchy/Vandermonde RS(n=12, k=8) erasure code over standardized Int8 frames (+50% FEC).
|
|
330
|
+
Recovers 100% of any <=4 dropped packets per 12-packet block; falls back to surviving
|
|
331
|
+
systematic packets + ZOH when >4 packets drop in a block.
|
|
332
|
+
"""
|
|
333
|
+
|
|
334
|
+
def __init__(self, n_channels: int, obs_mean: np.ndarray, obs_std: np.ndarray, k_data: int = 8, m_parity: int = 4) -> None:
|
|
335
|
+
self.n_channels = n_channels
|
|
336
|
+
self.mean = np.asarray(obs_mean, dtype=np.float64)
|
|
337
|
+
self.std = np.maximum(np.asarray(obs_std, dtype=np.float64), 1e-6)
|
|
338
|
+
self.k_data = k_data
|
|
339
|
+
self.m_parity = m_parity
|
|
340
|
+
self.n_total = k_data + m_parity
|
|
341
|
+
|
|
342
|
+
# Build systematic generator G of shape (n_total, k_data) over GF(256) using Cauchy matrix
|
|
343
|
+
g_sys = np.zeros((self.n_total, k_data), dtype=np.uint8)
|
|
344
|
+
g_sys[:k_data, :k_data] = np.eye(k_data, dtype=np.uint8)
|
|
345
|
+
for r in range(m_parity):
|
|
346
|
+
for c in range(k_data):
|
|
347
|
+
g_sys[k_data + r, c] = _GF.inv((r + 1) ^ (c + 16))
|
|
348
|
+
self.gen = g_sys
|
|
349
|
+
|
|
350
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray) -> Tuple[np.ndarray, int, int]:
|
|
351
|
+
t_steps, n = x_seq.shape
|
|
352
|
+
rx = np.zeros_like(x_seq, dtype=np.float64)
|
|
353
|
+
last = self.mean.copy()
|
|
354
|
+
total_bytes = 0
|
|
355
|
+
tx_pkts = 0
|
|
356
|
+
|
|
357
|
+
pkt_idx_global = 0
|
|
358
|
+
n_drop_len = len(dropped)
|
|
359
|
+
|
|
360
|
+
for start in range(0, t_steps, self.k_data):
|
|
361
|
+
end = min(t_steps, start + self.k_data)
|
|
362
|
+
actual_k = end - start
|
|
363
|
+
blk = np.zeros((self.k_data, n), dtype=np.uint8)
|
|
364
|
+
for j in range(actual_k):
|
|
365
|
+
z_norm = (x_seq[start + j] - self.mean) / self.std
|
|
366
|
+
q_i8 = np.clip(np.round(z_norm * (127.0 / 3.2)), -127, 127).astype(np.int8)
|
|
367
|
+
blk[j] = q_i8.view(np.uint8)
|
|
368
|
+
|
|
369
|
+
coded = _GF.matmul(self.gen, blk) # (12, N) uint8
|
|
370
|
+
arrived_rows: List[int] = []
|
|
371
|
+
for p_i in range(self.n_total):
|
|
372
|
+
# 3B FEC header (block_id + shard_idx) + N bytes
|
|
373
|
+
total_bytes += (3 + n)
|
|
374
|
+
tx_pkts += 1
|
|
375
|
+
is_lost = bool(dropped[pkt_idx_global % n_drop_len])
|
|
376
|
+
pkt_idx_global += 1
|
|
377
|
+
if not is_lost:
|
|
378
|
+
arrived_rows.append(p_i)
|
|
379
|
+
|
|
380
|
+
if len(arrived_rows) >= self.k_data:
|
|
381
|
+
use_rows = arrived_rows[: self.k_data]
|
|
382
|
+
sub_g = self.gen[use_rows]
|
|
383
|
+
inv_g = _GF.invert_matrix(sub_g)
|
|
384
|
+
rec_u8 = _GF.matmul(inv_g, coded[use_rows])
|
|
385
|
+
for j in range(actual_k):
|
|
386
|
+
q_rec = rec_u8[j].view(np.int8).astype(np.float64)
|
|
387
|
+
val = (q_rec * (3.2 / 127.0)) * self.std + self.mean
|
|
388
|
+
rx[start + j] = val
|
|
389
|
+
last = val.copy()
|
|
390
|
+
else:
|
|
391
|
+
# Beyond FEC cliff (>33% loss in block): salvage surviving systematic shards, ZOH lost ones
|
|
392
|
+
arrived_set = set(arrived_rows)
|
|
393
|
+
for j in range(actual_k):
|
|
394
|
+
if j in arrived_set:
|
|
395
|
+
q_rec = coded[j].view(np.int8).astype(np.float64)
|
|
396
|
+
last = (q_rec * (3.2 / 127.0)) * self.std + self.mean
|
|
397
|
+
rx[start + j] = last
|
|
398
|
+
|
|
399
|
+
return rx, total_bytes, tx_pkts
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
# ---------------------------------------------------------------------------
|
|
403
|
+
# Baseline 6: Reliable TCP / ARQ Retransmission Evaluator (Latency & Energy Cost)
|
|
404
|
+
# ---------------------------------------------------------------------------
|
|
405
|
+
|
|
406
|
+
class TCPARQAnalyticalEvaluator:
|
|
407
|
+
"""
|
|
408
|
+
Simulates reliable in-order ARQ / TCP delivery over a lossy link with round-trip time `rtt_ms`.
|
|
409
|
+
Reconstructs Int8 quantized values faithfully once delivered, but suffers:
|
|
410
|
+
1. Retransmission + ACK wire byte inflation: bytes * (1 + ack_ratio) / (1 - p_loss_eff)
|
|
411
|
+
2. Head-of-line blocking stall latency: when packet t requires k retries, all samples
|
|
412
|
+
t..t+delay are stalled on the receiver (evaluated both as stall_ms_p99 and real-time ZOH RMSE!).
|
|
413
|
+
"""
|
|
414
|
+
|
|
415
|
+
def __init__(self, n_channels: int, obs_mean: np.ndarray, obs_std: np.ndarray, rtt_ms: float, sample_dt_ms: float) -> None:
|
|
416
|
+
self.n_channels = n_channels
|
|
417
|
+
self.mean = np.asarray(obs_mean, dtype=np.float64)
|
|
418
|
+
self.std = np.maximum(np.asarray(obs_std, dtype=np.float64), 1e-6)
|
|
419
|
+
self.rtt_ms = rtt_ms
|
|
420
|
+
self.sample_dt_ms = sample_dt_ms
|
|
421
|
+
|
|
422
|
+
def run_stream(self, x_seq: np.ndarray, dropped: np.ndarray, seed: int = 123) -> Dict[str, object]:
|
|
423
|
+
rng = np.random.default_rng(seed)
|
|
424
|
+
t_steps, n = x_seq.shape
|
|
425
|
+
emp_loss = min(0.88, float(np.mean(dropped)))
|
|
426
|
+
|
|
427
|
+
# Quantize frames to Int8 + 12B compact TCP/MQTT framing overhead
|
|
428
|
+
q_frames = np.clip(np.round(((x_seq - self.mean) / self.std) * (127.0 / 3.2)), -127, 127).astype(np.int8)
|
|
429
|
+
dec_frames = (q_frames.astype(np.float64) * (3.2 / 127.0)) * self.std + self.mean
|
|
430
|
+
|
|
431
|
+
total_wire_bytes = 0
|
|
432
|
+
total_tx_pkts = 0
|
|
433
|
+
delivery_time_ms = np.zeros(t_steps, dtype=np.float64)
|
|
434
|
+
ho_line_time_ms = 0.0
|
|
435
|
+
|
|
436
|
+
for t in range(t_steps):
|
|
437
|
+
send_t_ms = t * self.sample_dt_ms
|
|
438
|
+
retries = 0
|
|
439
|
+
# First attempt uses `dropped[t]`; subsequent retries experience channel loss `emp_loss`
|
|
440
|
+
lost = bool(dropped[t])
|
|
441
|
+
while lost and retries < 8:
|
|
442
|
+
retries += 1
|
|
443
|
+
lost = bool(rng.random() < emp_loss)
|
|
444
|
+
|
|
445
|
+
attempts = retries + 1
|
|
446
|
+
# Forward payload (N + 12B header) * attempts + 8B ACK
|
|
447
|
+
total_wire_bytes += attempts * (n + 12) + 8
|
|
448
|
+
total_tx_pkts += attempts
|
|
449
|
+
arr_t_ms = send_t_ms + 0.5 * self.rtt_ms + retries * self.rtt_ms
|
|
450
|
+
# In-order TCP head-of-line blocking: packet t cannot surface before packet t-1!
|
|
451
|
+
ho_line_time_ms = max(ho_line_time_ms, arr_t_ms)
|
|
452
|
+
delivery_time_ms[t] = ho_line_time_ms
|
|
453
|
+
|
|
454
|
+
stalls_ms = delivery_time_ms - (np.arange(t_steps) * self.sample_dt_ms + 0.5 * self.rtt_ms)
|
|
455
|
+
# Real-time control/monitoring output at wall-clock time `t * sample_dt_ms + 0.5 * rtt_ms`:
|
|
456
|
+
rt_rx = np.zeros_like(x_seq, dtype=np.float64)
|
|
457
|
+
latest_avail = self.mean.copy()
|
|
458
|
+
avail_ptr = 0
|
|
459
|
+
for t in range(t_steps):
|
|
460
|
+
now_ms = t * self.sample_dt_ms + 0.5 * self.rtt_ms
|
|
461
|
+
while avail_ptr <= t and delivery_time_ms[avail_ptr] <= now_ms + 1e-9:
|
|
462
|
+
latest_avail = dec_frames[avail_ptr]
|
|
463
|
+
avail_ptr += 1
|
|
464
|
+
rt_rx[t] = latest_avail
|
|
465
|
+
|
|
466
|
+
return {
|
|
467
|
+
"realtime_rx": rt_rx,
|
|
468
|
+
"total_bytes": int(total_wire_bytes),
|
|
469
|
+
"tx_pkts": int(total_tx_pkts),
|
|
470
|
+
"mean_stall_ms": float(np.mean(stalls_ms)),
|
|
471
|
+
"p99_stall_ms": float(np.percentile(stalls_ms, 99)),
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
# ---------------------------------------------------------------------------
|
|
476
|
+
# Commercial Satellite / LoRaWAN / Cellular Cost & Battery Calculators
|
|
477
|
+
# ---------------------------------------------------------------------------
|
|
478
|
+
|
|
479
|
+
def compute_link_economics(
|
|
480
|
+
total_payload_bytes: int,
|
|
481
|
+
tx_packets: int,
|
|
482
|
+
duration_seconds: float,
|
|
483
|
+
link_type: str,
|
|
484
|
+
) -> Dict[str, float]:
|
|
485
|
+
"""
|
|
486
|
+
Computes annualized dollar airtime cost and radio TX energy consumption from
|
|
487
|
+
measured wire payload bytes and packet counts.
|
|
488
|
+
"""
|
|
489
|
+
sec_per_year = 365.25 * 86400.0
|
|
490
|
+
scale_to_year = sec_per_year / max(duration_seconds, 1e-6)
|
|
491
|
+
bytes_per_year = total_payload_bytes * scale_to_year
|
|
492
|
+
pkts_per_year = tx_packets * scale_to_year
|
|
493
|
+
|
|
494
|
+
if link_type == "iridium_sbd_satellite":
|
|
495
|
+
# Commercial Iridium SBD metered rate: $0.08 per 50-byte SBD credit (min 1 credit/msg)
|
|
496
|
+
# We evaluate continuous 50-byte burst packing over 10-second buoy reporting windows
|
|
497
|
+
windows_per_year = sec_per_year / 10.0
|
|
498
|
+
avg_bytes_per_10s = bytes_per_year / windows_per_year
|
|
499
|
+
if avg_bytes_per_10s < 1.0:
|
|
500
|
+
credits_per_year = avg_bytes_per_10s * windows_per_year / 50.0
|
|
501
|
+
else:
|
|
502
|
+
credits_per_10s = max(1.0, np.ceil(avg_bytes_per_10s / 50.0))
|
|
503
|
+
# Fraction of 10s windows that actually have >=1 transmitted packet
|
|
504
|
+
active_win_frac = min(1.0, (tx_packets / max(1.0, duration_seconds / 10.0)))
|
|
505
|
+
credits_per_year = windows_per_year * active_win_frac * credits_per_10s
|
|
506
|
+
annual_usd = float(credits_per_year * 0.08)
|
|
507
|
+
# Iridium 9603 transceiver: ~1.5W for ~0.35s per 50B burst = 0.525 J / credit
|
|
508
|
+
tx_wh_per_year = float((credits_per_year * 0.525) / 3600.0)
|
|
509
|
+
return {
|
|
510
|
+
"annual_cost_usd_per_node": round(annual_usd, 2),
|
|
511
|
+
"annual_fleet_100_nodes_usd": round(annual_usd * 100.0, 2),
|
|
512
|
+
"annual_radio_tx_wh": round(tx_wh_per_year, 2),
|
|
513
|
+
"bytes_per_day": round(bytes_per_year / 365.25, 1),
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
if link_type == "lorawan_sf9":
|
|
517
|
+
# Semtech SX1276 SF9/125kHz: ~4.1 ms per byte + 13B LoRa MAC header when a packet is sent
|
|
518
|
+
# Sub-packets within a 2s window coalesce into 1 LoRaWAN frame up to 120B
|
|
519
|
+
windows_2s = max(1.0, duration_seconds / 2.0)
|
|
520
|
+
active_frames = min(float(tx_packets), windows_2s)
|
|
521
|
+
total_air_bytes = total_payload_bytes + 13.0 * active_frames
|
|
522
|
+
airtime_s = (total_air_bytes * 0.0041)
|
|
523
|
+
duty_cycle_pct = 100.0 * (airtime_s / max(duration_seconds, 1e-6))
|
|
524
|
+
# Max nodes supported on a single 1%-duty-cycle LoRaWAN channel
|
|
525
|
+
max_nodes_1pct_channel = float(1.0 / max(duty_cycle_pct, 1e-6))
|
|
526
|
+
# SX1276 @ +14dBm TX: 44mA * 3.3V = 0.1452 W -> Wh/year
|
|
527
|
+
tx_wh_per_year = float((airtime_s * scale_to_year * 0.1452) / 3600.0)
|
|
528
|
+
# AA Li-SOCl2 cell capacity = 8.64 Wh (allocate 6.0 Wh to radio TX)
|
|
529
|
+
battery_years = min(20.0, 6.0 / max(tx_wh_per_year, 1e-4))
|
|
530
|
+
# Commercial LoRaWAN + Cloud IoT Core metered cost ($1.25 / MB + $1.00 / million msgs) across 1,000-node city AQMS
|
|
531
|
+
mb_per_year = bytes_per_year / 1e6
|
|
532
|
+
annual_usd_node = float(mb_per_year * 1.25 + (pkts_per_year / 1e6) * 1.00)
|
|
533
|
+
return {
|
|
534
|
+
"duty_cycle_pct": round(duty_cycle_pct, 5),
|
|
535
|
+
"max_nodes_per_1pct_channel": round(max_nodes_1pct_channel, 1),
|
|
536
|
+
"aa_battery_life_years": round(battery_years, 2),
|
|
537
|
+
"annual_fleet_1000_nodes_usd": round(annual_usd_node * 1000.0, 2),
|
|
538
|
+
"bytes_per_day": round(bytes_per_year / 365.25, 1),
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
# Cellular / Edge Wi-Fi / 5G metered M2M ($4.00 / GB rural 2G/M2M + radio TX 0.8W @ 250 kbps)
|
|
542
|
+
gb_per_year = bytes_per_year / 1e9
|
|
543
|
+
annual_usd_node = float(gb_per_year * 4.00)
|
|
544
|
+
airtime_s_yr = (bytes_per_year * 8.0) / 250_000.0
|
|
545
|
+
tx_wh_per_year = float((airtime_s_yr * 0.80) / 3600.0)
|
|
546
|
+
return {
|
|
547
|
+
"annual_cost_usd_per_node": round(annual_usd_node, 2),
|
|
548
|
+
"annual_fleet_1000_nodes_usd": round(annual_usd_node * 1000.0, 2),
|
|
549
|
+
"annual_radio_tx_wh": round(tx_wh_per_year, 2),
|
|
550
|
+
"bytes_per_day": round(bytes_per_year / 365.25, 1),
|
|
551
|
+
}
|