packbin 0.1.13__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- packbin-0.2.2/PKG-INFO +1100 -0
- packbin-0.2.2/README.md +1092 -0
- {packbin-0.1.13 → packbin-0.2.2}/pyproject.toml +1 -1
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_nodes.py +14 -4
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_unpack.py +52 -38
- packbin-0.2.2/src/packbin.egg-info/PKG-INFO +1100 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin.egg-info/SOURCES.txt +5 -1
- packbin-0.2.2/tests/test_bool_placement.py +159 -0
- packbin-0.2.2/tests/test_hostile_unpack.py +94 -0
- packbin-0.2.2/tests/test_hostile_vectors.py +41 -0
- packbin-0.2.2/tests/test_zero_width_elements.py +67 -0
- packbin-0.1.13/PKG-INFO +0 -650
- packbin-0.1.13/README.md +0 -642
- packbin-0.1.13/src/packbin.egg-info/PKG-INFO +0 -650
- {packbin-0.1.13 → packbin-0.2.2}/setup.cfg +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/__init__.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_errors.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_pack.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_scheme.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_session.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin/_session_pad.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin.egg-info/dependency_links.txt +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/src/packbin.egg-info/top_level.txt +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_binding.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_borrowed_count.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_fields.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_position.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_scheme.py +0 -0
- {packbin-0.1.13 → packbin-0.2.2}/tests/test_session.py +0 -0
packbin-0.2.2/PKG-INFO
ADDED
|
@@ -0,0 +1,1100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: packbin
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Pack and unpack a caller-owned field list
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
|
|
9
|
+
# packbin
|
|
10
|
+
|
|
11
|
+
[](https://www.npmjs.com/package/packbin)
|
|
12
|
+
[](https://www.nuget.org/packages/Packbin)
|
|
13
|
+
[](https://pypi.org/project/packbin/)
|
|
14
|
+
[](https://crates.io/crates/packbin)
|
|
15
|
+
[](https://central.sonatype.com/artifact/io.github.zxsanny/packbin)
|
|
16
|
+
[](LICENSE)
|
|
17
|
+
|
|
18
|
+
[)](https://github.com/zxsanny/packbin/actions/workflows/test.yml?query=branch%3Adev)
|
|
19
|
+
[)](https://github.com/zxsanny/packbin/actions/workflows/test.yml?query=branch%3Amain)
|
|
20
|
+
[](https://github.com/zxsanny/packbin/actions/workflows/publish.yml)
|
|
21
|
+
|
|
22
|
+
Binary packing and unpacking across languages with an optional encryption, declarative mapping, and zero overhead in the binary data.
|
|
23
|
+
|
|
24
|
+
Both sides keep the same field list. The bytes are only the values. Unpack takes the buffer and handlers. The first byte selects the handler, and that handler's scheme reads the rest.
|
|
25
|
+
|
|
26
|
+
Can be used for WebSocket, TCP, UDP, and other means of efficient communication
|
|
27
|
+
|
|
28
|
+
Encryption is optional. `PackSession` hides the packed bytes on one connection and adds 0 bytes to each packet. The only extra send is 16 bytes, once, when the connection opens. It does not detect a changed byte, and anyone holding the 32-byte seed can read every session. See [Encrypted session](#encrypted-session).
|
|
29
|
+
|
|
30
|
+
Java: Java 17+, Android API 26+.
|
|
31
|
+
|
|
32
|
+
## Example
|
|
33
|
+
|
|
34
|
+
Python → binary → TypeScript
|
|
35
|
+
|
|
36
|
+
### Python
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from packbin import BinaryPacker, Scheme, flags, i16, i32, u8, u16, utf8
|
|
40
|
+
|
|
41
|
+
class Position:
|
|
42
|
+
def __init__(self):
|
|
43
|
+
self.sid = 1
|
|
44
|
+
self.lat = 500_000_000
|
|
45
|
+
self.lon = 300_000_000
|
|
46
|
+
self.profile = 1
|
|
47
|
+
self.heading = None
|
|
48
|
+
self.speed = None
|
|
49
|
+
self.altitude = None
|
|
50
|
+
|
|
51
|
+
target = Scheme(
|
|
52
|
+
0x40,
|
|
53
|
+
Position,
|
|
54
|
+
u16(0, lambda row: row.sid),
|
|
55
|
+
i32(1, lambda row: row.lat),
|
|
56
|
+
i32(2, lambda row: row.lon),
|
|
57
|
+
u8(3, lambda row: row.profile),
|
|
58
|
+
flags(
|
|
59
|
+
4,
|
|
60
|
+
u16(4, lambda row: row.heading),
|
|
61
|
+
u8(5, lambda row: row.speed),
|
|
62
|
+
i16(6, lambda row: row.altitude),
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
raw = BinaryPacker.pack(target, Position())
|
|
67
|
+
|
|
68
|
+
class Ping:
|
|
69
|
+
def __init__(self):
|
|
70
|
+
self.code = 0
|
|
71
|
+
|
|
72
|
+
class Note:
|
|
73
|
+
def __init__(self):
|
|
74
|
+
self.id = 0
|
|
75
|
+
self.title = ""
|
|
76
|
+
|
|
77
|
+
ping = Scheme(2, Ping, u8(0, lambda row: row.code))
|
|
78
|
+
note = Scheme(3, Note, u16(0, lambda row: row.id), utf8(1, lambda row: row.title))
|
|
79
|
+
|
|
80
|
+
got = []
|
|
81
|
+
pinged = []
|
|
82
|
+
noted = []
|
|
83
|
+
BinaryPacker.unpack(raw, target.on(got.append), ping.on(pinged.append), note.on(noted.append))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
40 01 00 00 65 cd 1d 00 a3 e1 11 01 00
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### TypeScript
|
|
91
|
+
|
|
92
|
+
```ts
|
|
93
|
+
import { BinaryPacker, flags, i16, i32, scheme, u8, u16, utf8 } from "packbin"
|
|
94
|
+
|
|
95
|
+
class Target {
|
|
96
|
+
sid = 1
|
|
97
|
+
lat = 500_000_000
|
|
98
|
+
lon = 300_000_000
|
|
99
|
+
profile = 1
|
|
100
|
+
heading: number | null = null
|
|
101
|
+
speed: number | null = null
|
|
102
|
+
altitude: number | null = null
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const target = scheme<Target>(
|
|
106
|
+
0x40,
|
|
107
|
+
u16(0, (x) => x.sid),
|
|
108
|
+
i32(1, (x) => x.lat),
|
|
109
|
+
i32(2, (x) => x.lon),
|
|
110
|
+
u8(3, (x) => x.profile),
|
|
111
|
+
flags(4, [
|
|
112
|
+
u16(4, (x) => x.heading),
|
|
113
|
+
u8(5, (x) => x.speed),
|
|
114
|
+
i16(6, (x) => x.altitude),
|
|
115
|
+
]),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
class Ping {
|
|
119
|
+
code = 0
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
class Note {
|
|
123
|
+
id = 0
|
|
124
|
+
title = ""
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const ping = scheme<Ping>(2, u8(0, (x) => x.code))
|
|
128
|
+
const note = scheme<Note>(3, u16(0, (x) => x.id), utf8(1, (x) => x.title))
|
|
129
|
+
|
|
130
|
+
let got: Target | undefined
|
|
131
|
+
let pinged: Ping | undefined
|
|
132
|
+
let noted: Note | undefined
|
|
133
|
+
BinaryPacker.unpack(
|
|
134
|
+
raw,
|
|
135
|
+
target.on((row) => {
|
|
136
|
+
got = row
|
|
137
|
+
}),
|
|
138
|
+
ping.on((row) => {
|
|
139
|
+
pinged = row
|
|
140
|
+
}),
|
|
141
|
+
note.on((row) => {
|
|
142
|
+
noted = row
|
|
143
|
+
}),
|
|
144
|
+
)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`flags` is how an optional field takes no space when you have no value for it. `heading`, `speed`, and `altitude` are measurements, so `0` is still a value and has to be written. `None` means the field is not in the packet.
|
|
148
|
+
|
|
149
|
+
`motion` is always one byte in front of those fields. Bit 0 is `heading`, bit 1 is `speed`, bit 2 is `altitude`. A set bit writes that field next. A clear bit skips it.
|
|
150
|
+
|
|
151
|
+
| Field | Type | Bytes when present | Values |
|
|
152
|
+
|---|---|---|---|
|
|
153
|
+
| `heading` | `u16` | 2 | 0 … 65535 |
|
|
154
|
+
| `speed` | `u8` | 1 | 0 … 255 |
|
|
155
|
+
| `altitude` | `i16` | 2 | −32768 … 32767 |
|
|
156
|
+
|
|
157
|
+
In this example all three are `None`, so `motion` is `00` and those 5 bytes are absent. The packet is 13 bytes: type number 1, `sid` 2, `lat` 4, `lon` 4, `profile` 1, `motion` 1.
|
|
158
|
+
|
|
159
|
+
```
|
|
160
|
+
40 type
|
|
161
|
+
01 00 sid
|
|
162
|
+
00 65 cd 1d lat
|
|
163
|
+
00 a3 e1 11 lon
|
|
164
|
+
01 profile
|
|
165
|
+
00 motion
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`heading = 90` sets bit 0, so `motion` is `01` and `5a 00` follows it. `heading = 0` sets the same bit and writes `00 00`. `speed = 10` sets bit 1 and writes one byte. The present fields are written in the order listed, and only those.
|
|
169
|
+
|
|
170
|
+
## Example
|
|
171
|
+
|
|
172
|
+
C# → binary → Rust
|
|
173
|
+
|
|
174
|
+
### C#
|
|
175
|
+
|
|
176
|
+
```csharp
|
|
177
|
+
using Packbin;
|
|
178
|
+
|
|
179
|
+
sealed class User
|
|
180
|
+
{
|
|
181
|
+
public string Username { get; set; } = "";
|
|
182
|
+
public List<Role> Roles { get; set; } = [];
|
|
183
|
+
public Dictionary<string, ActionList> Access { get; set; } = [];
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
sealed class Role { public string RoleName { get; set; } = ""; }
|
|
187
|
+
sealed class ActionName { public string Action { get; set; } = ""; }
|
|
188
|
+
sealed class ActionList { public List<ActionName> Actions { get; set; } = []; }
|
|
189
|
+
|
|
190
|
+
var userScheme = new Scheme<User>(1, f => [
|
|
191
|
+
f.Utf8(0, x => x.Username),
|
|
192
|
+
f.List(x => x.Roles, r => r.Utf8(0, role => role.RoleName)),
|
|
193
|
+
f.Dict(x => x.Access, e => e.List(a => a.Actions, n => n.Utf8(0, action => action.Action)))]);
|
|
194
|
+
|
|
195
|
+
var raw = BinaryPacker.Pack(userScheme, new User
|
|
196
|
+
{
|
|
197
|
+
Username = "ada",
|
|
198
|
+
Roles = [new Role { RoleName = "user" }, new Role { RoleName = "admin" }],
|
|
199
|
+
Access = new()
|
|
200
|
+
{
|
|
201
|
+
["map"] = new ActionList { Actions = [new ActionName { Action = "read" }, new ActionName { Action = "edit" }] },
|
|
202
|
+
["store"] = new ActionList { Actions = [new ActionName { Action = "write" }] },
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
sealed class Ping { public byte Code { get; set; } }
|
|
207
|
+
sealed class Note
|
|
208
|
+
{
|
|
209
|
+
public ushort Id { get; set; }
|
|
210
|
+
public string Title { get; set; } = "";
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
var ping = new Scheme<Ping>(2, f => [f.U8(0, x => x.Code)]);
|
|
214
|
+
var note = new Scheme<Note>(3, f => [f.U16(0, x => x.Id), f.Utf8(1, x => x.Title)]);
|
|
215
|
+
|
|
216
|
+
User? got = null;
|
|
217
|
+
Ping? pinged = null;
|
|
218
|
+
Note? noted = null;
|
|
219
|
+
BinaryPacker.Unpack(
|
|
220
|
+
raw,
|
|
221
|
+
userScheme.On(row => got = row),
|
|
222
|
+
ping.On(row => pinged = row),
|
|
223
|
+
note.On(row => noted = row));
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
```
|
|
227
|
+
01 03 00 61 64 61 02 00 04 00 75 73 65 72 05 00 61 64 6d 69 6e
|
|
228
|
+
02 00 03 00 6d 61 70 02 00 04 00 72 65 61 64 04 00 65 64 69 74
|
|
229
|
+
05 00 73 74 6f 72 65 01 00 05 00 77 72 69 74 65
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
The first byte is the scheme type number.
|
|
233
|
+
|
|
234
|
+
### Rust
|
|
235
|
+
|
|
236
|
+
```rust
|
|
237
|
+
use packbin::{BinaryPacker, BoundField, Scheme};
|
|
238
|
+
use std::collections::BTreeMap;
|
|
239
|
+
|
|
240
|
+
#[derive(Default)]
|
|
241
|
+
struct User {
|
|
242
|
+
username: String,
|
|
243
|
+
roles: Vec<String>,
|
|
244
|
+
access: BTreeMap<String, Vec<String>>,
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
let user = Scheme::new(1, [
|
|
248
|
+
BoundField::utf8(
|
|
249
|
+
0,
|
|
250
|
+
|row: &User| row.username.clone(),
|
|
251
|
+
|row, value| row.username = value,
|
|
252
|
+
)
|
|
253
|
+
.into(),
|
|
254
|
+
BoundField::list_utf8(
|
|
255
|
+
|row: &User| row.roles.clone(),
|
|
256
|
+
|row, value| row.roles = value,
|
|
257
|
+
)
|
|
258
|
+
.into(),
|
|
259
|
+
BoundField::dict_list_utf8(
|
|
260
|
+
|row: &User| row.access.clone(),
|
|
261
|
+
|row, value| row.access = value,
|
|
262
|
+
)
|
|
263
|
+
.into(),
|
|
264
|
+
]);
|
|
265
|
+
|
|
266
|
+
#[derive(Default)]
|
|
267
|
+
struct Ping {
|
|
268
|
+
code: u8,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
#[derive(Default)]
|
|
272
|
+
struct Note {
|
|
273
|
+
id: u16,
|
|
274
|
+
title: String,
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
let ping = Scheme::new(2, [
|
|
278
|
+
BoundField::u8(0, |row: &Ping| row.code, |row, value| row.code = value).into(),
|
|
279
|
+
]);
|
|
280
|
+
|
|
281
|
+
let note = Scheme::new(3, [
|
|
282
|
+
BoundField::u16(0, |row: &Note| row.id, |row, value| row.id = value).into(),
|
|
283
|
+
BoundField::utf8(1, |row: &Note| row.title.clone(), |row, value| row.title = value).into(),
|
|
284
|
+
]);
|
|
285
|
+
|
|
286
|
+
let mut got = User::default();
|
|
287
|
+
let mut ping_row = Ping::default();
|
|
288
|
+
let mut note_row = Note::default();
|
|
289
|
+
BinaryPacker::unpack_with(
|
|
290
|
+
&raw,
|
|
291
|
+
&mut [
|
|
292
|
+
&mut user.on(|row| got = row),
|
|
293
|
+
&mut ping.on(|row| ping_row = row),
|
|
294
|
+
&mut note.on(|row| note_row = row),
|
|
295
|
+
],
|
|
296
|
+
)
|
|
297
|
+
.unwrap();
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
Schemes built from field lists (`MapScheme`) pack and unpack values keyed by field name with `packbin::pack(&scheme, &values)` and `packbin::unpack(&scheme, &bytes)`. That is the Rust path for the split flag-byte form, which the typed `Scheme` does not have yet. The typed `Scheme` has no `repeat` or `u2` either.
|
|
301
|
+
|
|
302
|
+
#### Times
|
|
303
|
+
|
|
304
|
+
A typed `times` binds a `Vec<E>` of element rows. `get` lends the elements, `set` takes the ones unpacked (one `E::default()` per round, filled by the members), and the members are items on `E`, numbered from the anchor on. The count is an earlier integer field of the row. Pack fails with a `PackError::Type` that names the `times` when the count differs from the number of elements: `times at id 1: count 3, 2 rounds`.
|
|
305
|
+
|
|
306
|
+
```rust
|
|
307
|
+
use packbin::{BinaryPacker, BoundField, Scheme, SchemeItem};
|
|
308
|
+
|
|
309
|
+
#[derive(Default, Debug, PartialEq)]
|
|
310
|
+
struct Point {
|
|
311
|
+
lat: i32,
|
|
312
|
+
lon: i32,
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
#[derive(Default, Debug, PartialEq)]
|
|
316
|
+
struct Route {
|
|
317
|
+
n: u8,
|
|
318
|
+
points: Vec<Point>,
|
|
319
|
+
tail: u8,
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
let route = Scheme::new(1, [
|
|
323
|
+
BoundField::u8(0, |row: &Route| row.n, |row, value| row.n = value).into(),
|
|
324
|
+
SchemeItem::times(
|
|
325
|
+
1,
|
|
326
|
+
0,
|
|
327
|
+
|row: &Route| &row.points[..],
|
|
328
|
+
|row, value| row.points = value,
|
|
329
|
+
[
|
|
330
|
+
BoundField::i32(1, |p: &Point| p.lat, |p, value| p.lat = value).into(),
|
|
331
|
+
BoundField::i32(2, |p: &Point| p.lon, |p, value| p.lon = value).into(),
|
|
332
|
+
],
|
|
333
|
+
),
|
|
334
|
+
BoundField::u8(3, |row: &Route| row.tail, |row, value| row.tail = value).into(),
|
|
335
|
+
]);
|
|
336
|
+
|
|
337
|
+
let row = Route {
|
|
338
|
+
n: 2,
|
|
339
|
+
points: vec![Point { lat: 10, lon: 20 }, Point { lat: 30, lon: 40 }],
|
|
340
|
+
tail: 7,
|
|
341
|
+
};
|
|
342
|
+
let raw = BinaryPacker::pack(&route, &row).unwrap();
|
|
343
|
+
|
|
344
|
+
let mut got = Route::default();
|
|
345
|
+
BinaryPacker::unpack_with(&raw, &mut [&mut route.on(|row| got = row)]).unwrap();
|
|
346
|
+
assert_eq!(got, row);
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
```
|
|
350
|
+
01 02 0a 00 00 00 14 00 00 00 1e 00 00 00 28 00 00 00 07
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
An optional member of `E` (a `BoundField::opt_u16` under `SchemeItem::flags`, or a `SchemeItem::when`) keeps its own round. The three-argument form `SchemeItem::times(anchor, count_id, members)` and the `SchemeItem::Times { .. }` struct variant are removed. The old form bound each member to one scalar of the row, so it lost every round after the first.
|
|
354
|
+
|
|
355
|
+
In a `MapScheme`, `times` unpacks to the per-name lists and also to the rounds, as `Value::Groups` under `__times_<anchor>` (present, and empty for a count of 0). A list holds only the rounds that read its name, so the rounds tell which round a value belongs to. `repeat` gives its rounds under `__repeat__`. Pack reads the rounds, and it fails with `PackError::Type("times at id 1: list for '1' disagrees with its rounds")` when a per-name list differs from them. To change a value, edit the rounds and drop or rewrite the per-name lists, or edit the lists and remove the `__times_<anchor>` key.
|
|
356
|
+
|
|
357
|
+
### C++
|
|
358
|
+
|
|
359
|
+
The C++ package packs into a buffer you own and reads from one. It throws nothing and allocates nothing, so the same code builds for a server and for a microcontroller (`-fno-exceptions -fno-rtti`). Strings and lists land in caller storage: `View` borrows bytes from the input buffer, `Text<N>` and `Array<T, N>` hold up to N, and more than N is the error `TooMany`, never a cut.
|
|
360
|
+
|
|
361
|
+
```cpp
|
|
362
|
+
#include <packbin/packbin.hpp>
|
|
363
|
+
|
|
364
|
+
using namespace packbin;
|
|
365
|
+
|
|
366
|
+
using Actions = Array<View, 8>;
|
|
367
|
+
|
|
368
|
+
struct User {
|
|
369
|
+
View username;
|
|
370
|
+
Array<View, 8> roles;
|
|
371
|
+
Array<Entry<Actions>, 8> access; // keys in unsigned byte order
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
struct Ping { std::uint8_t code = 0; };
|
|
375
|
+
struct Note { std::uint16_t id = 0; Text<32> title; };
|
|
376
|
+
|
|
377
|
+
constexpr auto user = scheme<User>(1,
|
|
378
|
+
utf8<&User::username>(0),
|
|
379
|
+
list<&User::roles>(utf8(0)),
|
|
380
|
+
dict<&User::access>(list(utf8(0))));
|
|
381
|
+
constexpr auto ping = scheme<Ping>(2, u8<&Ping::code>(0));
|
|
382
|
+
constexpr auto note = scheme<Note>(3, u16<&Note::id>(0), utf8<&Note::title>(1));
|
|
383
|
+
|
|
384
|
+
User row; // fill it like the C# example: "ada", two roles, map and store
|
|
385
|
+
std::uint8_t buf[128];
|
|
386
|
+
Result packed = pack(user, row, buf, sizeof(buf)); // packed.offset is the packet length
|
|
387
|
+
|
|
388
|
+
User got;
|
|
389
|
+
Ping pinged;
|
|
390
|
+
Note noted;
|
|
391
|
+
Result r = unpack(buf, packed.offset,
|
|
392
|
+
on(user, got, [](User const& u) { /* ... */ }),
|
|
393
|
+
on(ping, pinged, [](Ping const& p) { /* ... */ }),
|
|
394
|
+
on(note, noted, [](Note const& n) { /* ... */ }));
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
The first byte picks the scheme; its handler runs only after the whole packet reads. A failure is a value: `r.error` is one of `ShortPacket`, `TrailingBytes`, `TypeMismatch`, `BufferFull`, `TooMany`, `BadValue`, `SchemeInvalid`, with `r.offset` (the byte) and `r.field` (the order id). A `constexpr` scheme with a gap, a repeated id or a wrong anchor does not compile, and the compiler names the id; a scheme built at run time reports it through `validate(scheme)`. A member that may be absent is `Opt<T>`.
|
|
398
|
+
|
|
399
|
+
#### Microcontrollers
|
|
400
|
+
|
|
401
|
+
The same package builds for 32-bit microcontrollers — Cortex-M0+/M3/M4/M33, ESP32, RP2040, nRF52, STM32 — with `-std=c++17 -fno-exceptions -fno-rtti`. It references no `malloc`, no `operator new` and no exception runtime, keeps no global state, and the scheme tables live in flash.
|
|
402
|
+
|
|
403
|
+
| Tool | Add packbin |
|
|
404
|
+
|------|-------------|
|
|
405
|
+
| PlatformIO | `lib_deps = zxsanny/packbin` and `build_src_flags = -std=gnu++17` in `platformio.ini` |
|
|
406
|
+
| Arduino IDE / arduino-cli | Library Manager: `packbin`, then `#include <packbin.h>` |
|
|
407
|
+
| ESP-IDF ≥ 5.1 | `idf.py add-dependency "zxsanny/packbin"` |
|
|
408
|
+
| CMake / vcpkg (host) | vcpkg port `packbin`, or `add_subdirectory(cpp)` → target `packbin` |
|
|
409
|
+
|
|
410
|
+
Examples: [`cpp/examples/pico`](cpp/examples/pico) (PlatformIO, Raspberry Pi Pico), [`cpp/examples/esp32_arduino`](cpp/examples/esp32_arduino) (Arduino-ESP32), [`cpp/examples/esp_idf`](cpp/examples/esp_idf) (ESP-IDF with the hardware RNG).
|
|
411
|
+
|
|
412
|
+
`PackSession::start` takes the board's random source as a `RandomFn` (`bool(std::uint8_t* out, std::size_t n, void* ctx)`): `esp_fill_random` on ESP32, the RNG peripheral on STM32 or nRF52. When it returns false, no session opens. Host programs pass `packbin::os_random`. `f64` needs an 8-byte `double`; where `double` is 4 bytes (avr-gcc), a scheme with `f64` does not compile.
|
|
413
|
+
|
|
414
|
+
Moving from 0.1.x — the C++ API changed; the bytes did not:
|
|
415
|
+
|
|
416
|
+
| 0.1.x | Now |
|
|
417
|
+
|-------|-----|
|
|
418
|
+
| `BinaryPacker::pack(scheme, row)` → `std::vector` | `pack(scheme, row, out, cap)` → `Result` (`offset` = length) |
|
|
419
|
+
| `BinaryPacker::unpack(bytes, scheme.on(handler)...)` | `unpack(data, len, on(scheme, row, handler)...)`, or `unpack(scheme, data, len, row)` |
|
|
420
|
+
| `Scheme<T>(type, ...)`, `scheme(type, {...})`, `SchemeHandler` | `constexpr auto s = scheme<T>(type, ...)`; `on(s, row, handler)` |
|
|
421
|
+
| `Value`, `Values`, `ValueList`, `ValueMap` (map rows) | struct rows with `Opt<T>`, `View`, `Text<N>`, `Blob<N>`, `Array<T, N>`, `Entry<V>` members |
|
|
422
|
+
| `u16(0, &T::m)`, `bind_scalar`, `bind_optional`, `bind_accessors`, `BoundField<T>` | `u16<&T::m>(0)` (member binding only; no getter/setter) |
|
|
423
|
+
| `std::optional<T>` / `std::string` / `std::vector<T>` members | `Opt<T>` / `View` or `Text<N>` / `Array<T, N>` |
|
|
424
|
+
| `flags(id, {a, b})`, `when(id, eq, {a})`, `repeat(id, {a})`, `times(id, n, {a})`, `group(id, {a})` | the same without braces: `flags(id, a, b)` …; `repeat` and `times` bind an `Array<E, N>` member: `repeat<&T::m>(id, ...)` |
|
|
425
|
+
| `group(id, "name", {})` | `group<&T::flag>(id)` on a `bool` member |
|
|
426
|
+
| `flag_byte()`, `Field::bit(field)` | `flag_byte(n)`, `flag_bit(n, field)` with n = 0..7 |
|
|
427
|
+
| `list("name", element)`, `dict("name", element)` | `list<&T::m>(element)`, `dict<&T::m>(element)` |
|
|
428
|
+
| `u2({0, 1})`, `sized(id, n)`, `bits(id, n)`, `packed(w, id, n, bias)` | `u2(u8<&T::a>(0), u8<&T::b>(1))`, `sized<&T::m>(id, n)`, `bits<&T::m>(id, n)`, `packed<&T::m>(w, id, n, bias)` |
|
|
429
|
+
| `bytes(id, n)` with `std::vector<std::uint8_t>` | `bytes<&T::m>(id)` on `std::uint8_t[n]`, or `bytes<&T::m>(id, n)` on a `View` |
|
|
430
|
+
| `eq(id, Value{...})` | `eq(id, number)` |
|
|
431
|
+
| `UnpackResult`, `ShortPacket`, `TrailingBytes`, `TypeMismatch`, `std::runtime_error` | `Result` and `Error` |
|
|
432
|
+
| `validate_order(fields)` | `validate(scheme)`, or a compile error for a `constexpr` scheme |
|
|
433
|
+
| `to_hex`, `mismatched_bytes`, `present`, `motion_field_count`, `id_name`, `pack_body`, `unpack_body`, `Field` | removed |
|
|
434
|
+
| `PackSession::load(seed)` → `std::optional` | `PackSession s; s.load(seed, 32)` → `bool` |
|
|
435
|
+
| `start()`, `start(nonce)`, `join(nonce)` | `start(os_random, nullptr, nonce_out)` (or your `RandomFn`), `start(nonce, 16)`, `join(nonce, 16)` |
|
|
436
|
+
| `session.pack(scheme, row)` → `std::optional<std::vector>` | `session.pack(scheme, row, out, cap)` → `Result` |
|
|
437
|
+
| `session.unpack(bytes, handlers...)` | `session.unpack(data, len, ...)`; removes the pad in place |
|
|
438
|
+
|
|
439
|
+
Two kinds of scheme fail construction (`SchemeInvalid`, or a compile error for a `constexpr` scheme), because neither can round-trip: a `boolean` or an empty `group` anywhere except directly under `flags(...)` or `flag_bit(...)`, and a `u2` with more than 64 children. The wire bytes of every valid scheme are unchanged.
|
|
440
|
+
|
|
441
|
+
## Encrypted session
|
|
442
|
+
|
|
443
|
+
One optional session beside clear pack. Load a 32-byte seed on each side. The opener calls start and sends those 16 bytes once. The waiter calls join with them. After that, pack and unpack use the same schemes as clear pack, and the payload stays the same length. A second client is a second session.
|
|
444
|
+
|
|
445
|
+
The keys come from HKDF-SHA256 over the seed and the 16 bytes, one key per direction. Each packet is XORed with a ChaCha20 stream at the next counter. The counter is not sent, so the connection must deliver packets in order and drop none. Nothing checks that a byte was changed on the way. The seed never leaves the process. Replace it with a software update.
|
|
446
|
+
|
|
447
|
+
### C#
|
|
448
|
+
|
|
449
|
+
```csharp
|
|
450
|
+
using Packbin;
|
|
451
|
+
|
|
452
|
+
byte[] seed = /* 32 bytes shared out of band */;
|
|
453
|
+
|
|
454
|
+
var opener = PackSession.Load(seed)!;
|
|
455
|
+
byte[] nonce = opener.Start()!;
|
|
456
|
+
|
|
457
|
+
var waiter = PackSession.Load(seed)!;
|
|
458
|
+
waiter.Join(nonce);
|
|
459
|
+
|
|
460
|
+
sealed class Ping { public byte Code { get; set; } }
|
|
461
|
+
var ping = new Scheme<Ping>(2, f => [f.U8(0, x => x.Code)]);
|
|
462
|
+
|
|
463
|
+
var payload = opener.Pack(ping, new Ping { Code = 7 })!;
|
|
464
|
+
|
|
465
|
+
Ping? got = null;
|
|
466
|
+
waiter.Unpack(payload, ping.On(row => got = row));
|
|
467
|
+
```
|
|
468
|
+
|
|
469
|
+
The 16 bytes leave once. There is no second handshake on that connection.
|
|
470
|
+
|
|
471
|
+
### WebSocket: Vue client, C# server
|
|
472
|
+
|
|
473
|
+
The browser opens the socket, so it is the opener. Its first message is the 16 bytes. The server treats the first message on every socket as those bytes and keeps one session per socket. Every later message in either direction is one packed packet, encrypted.
|
|
474
|
+
|
|
475
|
+
Both sides share the schemes:
|
|
476
|
+
|
|
477
|
+
| Type | Scheme | Direction |
|
|
478
|
+
|------|--------|-----------|
|
|
479
|
+
| `0x40` | position: `sid` u16, `lat` i32, `lon` i32, `profile` u8 | client → server |
|
|
480
|
+
| `0x41` | ack: `sid` u16, `accepted` u8 | server → client |
|
|
481
|
+
|
|
482
|
+
#### Server — ASP.NET Core
|
|
483
|
+
|
|
484
|
+
```csharp
|
|
485
|
+
using System.Net.WebSockets;
|
|
486
|
+
using Packbin;
|
|
487
|
+
|
|
488
|
+
var builder = WebApplication.CreateBuilder(args);
|
|
489
|
+
var app = builder.Build();
|
|
490
|
+
|
|
491
|
+
byte[] seed = Convert.FromHexString(builder.Configuration["Packbin:Seed"]!); // 32 bytes
|
|
492
|
+
var position = new Scheme<Position>(0x40, f => [
|
|
493
|
+
f.U16(0, x => x.Sid),
|
|
494
|
+
f.I32(1, x => x.Lat),
|
|
495
|
+
f.I32(2, x => x.Lon),
|
|
496
|
+
f.U8(3, x => x.Profile)]);
|
|
497
|
+
var ack = new Scheme<Ack>(0x41, f => [
|
|
498
|
+
f.U16(0, x => x.Sid),
|
|
499
|
+
f.U8(1, x => x.Accepted)]);
|
|
500
|
+
|
|
501
|
+
app.UseWebSockets();
|
|
502
|
+
app.Map("/ws", async context =>
|
|
503
|
+
{
|
|
504
|
+
if (!context.WebSockets.IsWebSocketRequest)
|
|
505
|
+
{
|
|
506
|
+
context.Response.StatusCode = StatusCodes.Status400BadRequest;
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
using var socket = await context.WebSockets.AcceptWebSocketAsync();
|
|
511
|
+
var ct = context.RequestAborted;
|
|
512
|
+
var session = PackSession.Load(seed)!;
|
|
513
|
+
|
|
514
|
+
var nonce = await ReceiveAsync(socket, ct);
|
|
515
|
+
if (nonce is null || !session.Join(nonce))
|
|
516
|
+
{
|
|
517
|
+
await socket.CloseAsync(WebSocketCloseStatus.PolicyViolation, "handshake", ct);
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
while (await ReceiveAsync(socket, ct) is { } packet)
|
|
522
|
+
{
|
|
523
|
+
Position? got = null;
|
|
524
|
+
var err = session.Unpack(packet, position.On(row => got = row));
|
|
525
|
+
if (err is not null || got is null)
|
|
526
|
+
{
|
|
527
|
+
await socket.CloseAsync(WebSocketCloseStatus.InvalidPayloadData, "packet", ct);
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
var reply = session.Pack(ack, new Ack { Sid = got.Sid, Accepted = 1 })!;
|
|
532
|
+
await socket.SendAsync(reply, WebSocketMessageType.Binary, true, ct);
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
app.Run();
|
|
537
|
+
|
|
538
|
+
static async Task<byte[]?> ReceiveAsync(WebSocket socket, CancellationToken ct)
|
|
539
|
+
{
|
|
540
|
+
var buffer = new byte[4096];
|
|
541
|
+
using var message = new MemoryStream();
|
|
542
|
+
while (true)
|
|
543
|
+
{
|
|
544
|
+
var result = await socket.ReceiveAsync(buffer, ct);
|
|
545
|
+
if (result.MessageType == WebSocketMessageType.Close)
|
|
546
|
+
return null;
|
|
547
|
+
message.Write(buffer, 0, result.Count);
|
|
548
|
+
if (result.EndOfMessage)
|
|
549
|
+
return message.ToArray();
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
sealed class Position
|
|
554
|
+
{
|
|
555
|
+
public ushort Sid { get; set; }
|
|
556
|
+
public int Lat { get; set; }
|
|
557
|
+
public int Lon { get; set; }
|
|
558
|
+
public byte Profile { get; set; }
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
sealed class Ack
|
|
562
|
+
{
|
|
563
|
+
public ushort Sid { get; set; }
|
|
564
|
+
public byte Accepted { get; set; }
|
|
565
|
+
}
|
|
566
|
+
```
|
|
567
|
+
|
|
568
|
+
The `while` loop sends and receives on one task, so the send counter advances once per reply. If several tasks write to the same socket, send through one queue: the packets must leave in the order `Pack` numbered them.
|
|
569
|
+
|
|
570
|
+
#### Client — Vue 3, TypeScript
|
|
571
|
+
|
|
572
|
+
`src/live/packets.ts`:
|
|
573
|
+
|
|
574
|
+
```ts
|
|
575
|
+
import { i32, scheme, u16, u8 } from "packbin"
|
|
576
|
+
|
|
577
|
+
export class Position {
|
|
578
|
+
sid = 0
|
|
579
|
+
lat = 0
|
|
580
|
+
lon = 0
|
|
581
|
+
profile = 0
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
export class Ack {
|
|
585
|
+
sid = 0
|
|
586
|
+
accepted = 0
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
export const position = scheme<Position>(
|
|
590
|
+
0x40,
|
|
591
|
+
u16(0, (x) => x.sid),
|
|
592
|
+
i32(1, (x) => x.lat),
|
|
593
|
+
i32(2, (x) => x.lon),
|
|
594
|
+
u8(3, (x) => x.profile),
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
export const ack = scheme<Ack>(
|
|
598
|
+
0x41,
|
|
599
|
+
u16(0, (x) => x.sid),
|
|
600
|
+
u8(1, (x) => x.accepted),
|
|
601
|
+
)
|
|
602
|
+
```
|
|
603
|
+
|
|
604
|
+
`src/live/useLiveSocket.ts`:
|
|
605
|
+
|
|
606
|
+
```ts
|
|
607
|
+
import { onBeforeUnmount, ref } from "vue"
|
|
608
|
+
import { PackSession } from "packbin"
|
|
609
|
+
import { ack, position, type Ack, type Position } from "./packets"
|
|
610
|
+
|
|
611
|
+
const seed = Uint8Array.from(
|
|
612
|
+
import.meta.env.VITE_PACKBIN_SEED.match(/../g)!.map((h: string) => parseInt(h, 16)),
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
export function useLiveSocket(url: string) {
|
|
616
|
+
const lastAck = ref<Ack | null>(null)
|
|
617
|
+
const open = ref(false)
|
|
618
|
+
let socket: WebSocket | null = null
|
|
619
|
+
let session: PackSession | null = null
|
|
620
|
+
let retry = 0
|
|
621
|
+
|
|
622
|
+
function connect() {
|
|
623
|
+
session = PackSession.load(seed)
|
|
624
|
+
socket = new WebSocket(url)
|
|
625
|
+
socket.binaryType = "arraybuffer"
|
|
626
|
+
|
|
627
|
+
socket.onopen = () => {
|
|
628
|
+
socket!.send(session!.start()!)
|
|
629
|
+
open.value = true
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
socket.onmessage = (ev: MessageEvent<ArrayBuffer>) => {
|
|
633
|
+
const result = session!.unpack(ev.data, ack.on((row) => (lastAck.value = row)))
|
|
634
|
+
if (!result.ok) socket!.close(4000, "packet")
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
socket.onclose = () => {
|
|
638
|
+
open.value = false
|
|
639
|
+
socket = null
|
|
640
|
+
session = null
|
|
641
|
+
retry = window.setTimeout(connect, 1000)
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function sendPosition(row: Position) {
|
|
646
|
+
if (!open.value || !socket || !session) return false
|
|
647
|
+
socket.send(session.pack(position, row)!)
|
|
648
|
+
return true
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
connect()
|
|
652
|
+
onBeforeUnmount(() => {
|
|
653
|
+
window.clearTimeout(retry)
|
|
654
|
+
if (socket) {
|
|
655
|
+
socket.onclose = null
|
|
656
|
+
socket.close()
|
|
657
|
+
}
|
|
658
|
+
})
|
|
659
|
+
|
|
660
|
+
return { open, lastAck, sendPosition }
|
|
661
|
+
}
|
|
662
|
+
```
|
|
663
|
+
|
|
664
|
+
`src/components/LivePosition.vue`:
|
|
665
|
+
|
|
666
|
+
```vue
|
|
667
|
+
<script setup lang="ts">
|
|
668
|
+
import { useLiveSocket } from "../live/useLiveSocket"
|
|
669
|
+
|
|
670
|
+
const { open, lastAck, sendPosition } = useLiveSocket(
|
|
671
|
+
`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/ws`,
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
function send() {
|
|
675
|
+
sendPosition({ sid: 1, lat: 500_000_000, lon: 300_000_000, profile: 1 })
|
|
676
|
+
}
|
|
677
|
+
</script>
|
|
678
|
+
|
|
679
|
+
<template>
|
|
680
|
+
<button :disabled="!open" @click="send">Send position</button>
|
|
681
|
+
<p v-if="lastAck">Server accepted sid {{ lastAck.sid }}</p>
|
|
682
|
+
</template>
|
|
683
|
+
```
|
|
684
|
+
|
|
685
|
+
Each new socket needs a new session. `connect` loads a fresh one, because `start` and `join` erase the session's copy of the seed and a session opens only once. A reconnect sends new 16 bytes.
|
|
686
|
+
|
|
687
|
+
What travels for one position:
|
|
688
|
+
|
|
689
|
+
| Message | Bytes |
|
|
690
|
+
|---------|-------|
|
|
691
|
+
| Handshake, once per socket | 16 |
|
|
692
|
+
| Position, clear pack | 12 |
|
|
693
|
+
| Position, encrypted | 12 |
|
|
694
|
+
| Ack, encrypted | 4 |
|
|
695
|
+
|
|
696
|
+
`VITE_PACKBIN_SEED` is compiled into the JavaScript bundle, so anyone who downloads the app can read it. The session hides packets from a network observer. It does not hide them from a person who has the app. Change the seed with a release, and keep the old value on the server until clients have updated.
|
|
697
|
+
|
|
698
|
+
## Data types
|
|
699
|
+
|
|
700
|
+
| Helper | What it writes |
|
|
701
|
+
|--------|----------------|
|
|
702
|
+
| `u8` `u16` `u32` `u64` | unsigned integer, little-endian |
|
|
703
|
+
| `i8` `i16` `i32` `i64` | signed integer, little-endian |
|
|
704
|
+
| `f32` `f64` | IEEE 754 float |
|
|
705
|
+
| `bytes(n)` | exactly `n` raw bytes |
|
|
706
|
+
| `be(field)` | that number, big-endian |
|
|
707
|
+
| `utf8` | UTF-8 string, `u16` length |
|
|
708
|
+
| `list` | `u16` count, then that many elements |
|
|
709
|
+
| `dict` | `u16` pair count; keys in unsigned byte order |
|
|
710
|
+
| `flags(anchor, fields)` | one `u8`; bit 0 is the first field; a clear bit omits that field; at most 8 fields; the anchor is not written |
|
|
711
|
+
| `bool` | a flag bit with no payload; set only for `true`; allowed only directly under `flags` or a flag-byte bit |
|
|
712
|
+
| `when(anchor, eq(id, value), fields)` | the group only when an earlier field of the same scope equals `value`; the anchor is not written |
|
|
713
|
+
| `repeat(anchor, fields)` | the group until the buffer ends; no count; the anchor is not written |
|
|
714
|
+
| `sized` | raw bytes whose length is an earlier integer |
|
|
715
|
+
| `u2` | fixed 2-bit slots, low bits first |
|
|
716
|
+
| `bits` | 1-bit list; the count is an earlier integer |
|
|
717
|
+
| `packed` | 1-bit or 2-bit list; the count is an earlier integer plus a bias |
|
|
718
|
+
| `times(anchor, count, fields)` | the inner fields exactly N times, then the next field; the anchor is not written |
|
|
719
|
+
|
|
720
|
+
Field numbers are the order in the scheme, starting at 0. They are not written. A gap, a repeated id, or an anchor that is not the next value id fails construction. The anchor is not written, repeat still has no count, and a list, a dict, and a nested group still start at 0. The snippets below are Python. The other languages use the same order and produce the same bytes. The first byte of every packet is the scheme type number.
|
|
721
|
+
|
|
722
|
+
### Integers
|
|
723
|
+
|
|
724
|
+
Little-endian. `u16` 1 is `01 00`. `i16` −2 is `fe ff`.
|
|
725
|
+
|
|
726
|
+
```python
|
|
727
|
+
row = Scheme(1, dict, u16(0, lambda row: row["n"]), i16(1, lambda row: row["d"]))
|
|
728
|
+
BinaryPacker.pack(row, {"n": 1, "d": -2})
|
|
729
|
+
```
|
|
730
|
+
|
|
731
|
+
```
|
|
732
|
+
01 01 00 fe ff
|
|
733
|
+
```
|
|
734
|
+
|
|
735
|
+
`0` is a value and is written. A missing optional field is absence, which only `flags` can express.
|
|
736
|
+
|
|
737
|
+
Pack refuses a value that does not fit. TypeScript takes an integer `number` or a `bigint` and throws a `RangeError` that names the member: `n: 300 does not fit in u8` for a value out of range, a fraction or `NaN`, and `n: expected a number for u8, got string` for anything else. It used to wrap 300 to `2c` and cut 1.5 to 1. Pass a `bigint` for a 64-bit value: a `number` that is not a safe integer is refused even where the width could hold it (`n: 9007199254740992 does not fit in u64; pass a bigint`). A row that unpack returned packs again.
|
|
738
|
+
|
|
739
|
+
### Floats
|
|
740
|
+
|
|
741
|
+
IEEE 754, little-endian. `f32` 1.5 is `00 00 c0 3f`.
|
|
742
|
+
|
|
743
|
+
```python
|
|
744
|
+
row = Scheme(1, dict, f32(0, lambda row: row["x"]))
|
|
745
|
+
BinaryPacker.pack(row, {"x": 1.5})
|
|
746
|
+
```
|
|
747
|
+
|
|
748
|
+
```
|
|
749
|
+
01 00 00 c0 3f
|
|
750
|
+
```
|
|
751
|
+
|
|
752
|
+
`f64` is the same layout in eight bytes.
|
|
753
|
+
|
|
754
|
+
TypeScript pack takes only a `number` for a float. A `bigint` or a numeric string throws a `RangeError` (`x: expected a number for f64, got string`), and so does a finite value too large for `f32` (`x: 1e+39 does not fit in f32`). `NaN` and the infinities are written as given.
|
|
755
|
+
|
|
756
|
+
### Fixed bytes
|
|
757
|
+
|
|
758
|
+
`bytes(n)` writes exactly `n` bytes and no length. `b"abc"` with `n = 3` is the three characters.
|
|
759
|
+
|
|
760
|
+
```python
|
|
761
|
+
row = Scheme(1, dict, bytes(0, lambda row: row["b"], 3))
|
|
762
|
+
BinaryPacker.pack(row, {"b": b"abc"})
|
|
763
|
+
```
|
|
764
|
+
|
|
765
|
+
```
|
|
766
|
+
01 61 62 63
|
|
767
|
+
```
|
|
768
|
+
|
|
769
|
+
A shorter or longer value is rejected.
|
|
770
|
+
|
|
771
|
+
### Big-endian
|
|
772
|
+
|
|
773
|
+
`be` flips one number. The rest of the scheme stays little-endian. `u16` 1 becomes `00 01`.
|
|
774
|
+
|
|
775
|
+
```python
|
|
776
|
+
row = Scheme(1, dict, be(u16(0, lambda row: row["n"])))
|
|
777
|
+
BinaryPacker.pack(row, {"n": 1})
|
|
778
|
+
```
|
|
779
|
+
|
|
780
|
+
```
|
|
781
|
+
01 00 01
|
|
782
|
+
```
|
|
783
|
+
|
|
784
|
+
### UTF-8
|
|
785
|
+
|
|
786
|
+
A `u16` byte count, then the UTF-8 bytes. `"zxsanny"` is 7 bytes. An empty string is `00 00`. More than 65535 bytes is rejected.
|
|
787
|
+
|
|
788
|
+
```python
|
|
789
|
+
row = Scheme(1, dict, utf8(0, lambda row: row["name"]))
|
|
790
|
+
BinaryPacker.pack(row, {"name": "zxsanny"})
|
|
791
|
+
```
|
|
792
|
+
|
|
793
|
+
```
|
|
794
|
+
01 07 00 7a 78 73 61 6e 6e 79
|
|
795
|
+
```
|
|
796
|
+
|
|
797
|
+
### List
|
|
798
|
+
|
|
799
|
+
A `u16` count, then that many elements. The element is one field, and its id restarts at 0 inside the list. `[1, 2]` of `u16` is count 2, then `01 00`, then `02 00`. An empty list is `00 00` and still leaves the next scheme field readable.
|
|
800
|
+
|
|
801
|
+
```python
|
|
802
|
+
row = Scheme(1, dict, list(lambda row: row["xs"], u16(0, lambda row: row)))
|
|
803
|
+
BinaryPacker.pack(row, {"xs": [1, 2]})
|
|
804
|
+
```
|
|
805
|
+
|
|
806
|
+
```
|
|
807
|
+
01 02 00 01 00 02 00
|
|
808
|
+
```
|
|
809
|
+
|
|
810
|
+
### Dictionary
|
|
811
|
+
|
|
812
|
+
A `u16` pair count. Each pair is a UTF-8 key, then one element. Keys are written in unsigned byte order, so insert order does not change the packet. `"a"` then `"b"`:
|
|
813
|
+
|
|
814
|
+
```python
|
|
815
|
+
from packbin import dict as map_field
|
|
816
|
+
|
|
817
|
+
row = Scheme(1, dict, map_field(lambda row: row["m"], u8(0, lambda row: row)))
|
|
818
|
+
BinaryPacker.pack(row, {"m": {"b": 1, "a": 2}})
|
|
819
|
+
```
|
|
820
|
+
|
|
821
|
+
```
|
|
822
|
+
01 02 00 01 00 61 02 01 00 62 01
|
|
823
|
+
```
|
|
824
|
+
|
|
825
|
+
`02 00` is two pairs. `01 00 61` is the key `"a"`, then value `02`. `01 00 62` is `"b"`, then value `01`.
|
|
826
|
+
|
|
827
|
+
### Flags
|
|
828
|
+
|
|
829
|
+
One byte in front of the fields. Bit 0 is the first field. A set bit writes that field next. A clear bit skips it, so the field takes no space. `0` is still present: it sets the bit and writes a zero. `None` leaves the bit clear.
|
|
830
|
+
|
|
831
|
+
`heading = 90` sets bit 0 and writes `5a 00`. `speed` is absent, so bit 1 stays clear.
|
|
832
|
+
|
|
833
|
+
```python
|
|
834
|
+
row = Scheme(
|
|
835
|
+
1,
|
|
836
|
+
dict,
|
|
837
|
+
flags(
|
|
838
|
+
0,
|
|
839
|
+
u16(0, lambda row: row["heading"]),
|
|
840
|
+
u8(1, lambda row: row["speed"]),
|
|
841
|
+
),
|
|
842
|
+
)
|
|
843
|
+
BinaryPacker.pack(row, {"heading": 90})
|
|
844
|
+
```
|
|
845
|
+
|
|
846
|
+
```
|
|
847
|
+
01 01 5a 00
|
|
848
|
+
```
|
|
849
|
+
|
|
850
|
+
`group` gathers several fields under one bit. The bit is set when any of them is present, and those fields are written in order.
|
|
851
|
+
|
|
852
|
+
### Bool
|
|
853
|
+
|
|
854
|
+
A `bool` inside `flags` sets its bit and writes nothing after it. Here bit 0 is the bool and bit 1 is a `u8` of 7, so the flags byte is `03` and the only payload is `07`.
|
|
855
|
+
|
|
856
|
+
The bit is set only for `true`; `false` and a missing value leave it clear, and unpack gives `true` only when the bit is set. A `bool` anywhere except directly under `flags` or a flag-byte bit fails scheme construction, because it has no bit to live in. So does a ninth bit in one flags byte.
|
|
857
|
+
|
|
858
|
+
```python
|
|
859
|
+
row = Scheme(
|
|
860
|
+
1,
|
|
861
|
+
dict,
|
|
862
|
+
flags(
|
|
863
|
+
0,
|
|
864
|
+
bool(0, lambda row: row["on"]),
|
|
865
|
+
u8(1, lambda row: row["n"]),
|
|
866
|
+
),
|
|
867
|
+
)
|
|
868
|
+
BinaryPacker.pack(row, {"on": True, "n": 7})
|
|
869
|
+
```
|
|
870
|
+
|
|
871
|
+
```
|
|
872
|
+
01 03 07
|
|
873
|
+
```
|
|
874
|
+
|
|
875
|
+
A `when` that tests a bool matches only when the bit is set, so test it with `eq(id, true)`. A clear bit is absent when a packet is read, so `eq(id, false)` never matches. C# pack agrees. TypeScript pack and Rust pack (map form) still write the `when` body for `eq(id, false)` on a clear bit, and their own unpack then rejects the packet.
|
|
876
|
+
|
|
877
|
+
### When
|
|
878
|
+
|
|
879
|
+
The group is written only when an earlier field equals the given value. The tested field must already have been read, and it must be in the same scope. The top level, the body of a `repeat` or `times`, a `list` or `dict` element and a nested row are separate scopes, and a `when` sees only the fields before it in its own. A field that was skipped (inside a `when` that did not match, or behind a clear flag bit) is absent, so a `when` that tests it does not match when a packet is read. `profile == 0` writes `shape`. Any other profile writes nothing after the profile byte.
|
|
880
|
+
|
|
881
|
+
```python
|
|
882
|
+
row = Scheme(
|
|
883
|
+
1,
|
|
884
|
+
dict,
|
|
885
|
+
u8(0, lambda row: row["profile"]),
|
|
886
|
+
when(1, eq(0, 0), u8(1, lambda row: row["shape"])),
|
|
887
|
+
)
|
|
888
|
+
BinaryPacker.pack(row, {"profile": 0, "shape": 9})
|
|
889
|
+
```
|
|
890
|
+
|
|
891
|
+
```
|
|
892
|
+
01 00 09
|
|
893
|
+
```
|
|
894
|
+
|
|
895
|
+
`profile = 1` is `01 01`.
|
|
896
|
+
|
|
897
|
+
A scheme whose `when` names a later field, an id that does not exist, or a field of another scope fails when you build it (every package but Python; see [Untrusted input](#untrusted-input)). C# pack decides every `when` from the fields it wrote, as unpack does from the fields it read. TypeScript pack still tests the row you give it, so a `when` on a field that another `when` skipped can write a body its own unpack rejects. Do not test a field that something else can skip.
|
|
898
|
+
|
|
899
|
+
### Repeat
|
|
900
|
+
|
|
901
|
+
The group is repeated until the buffer ends. There is no count. Unpack stops on the last complete group. One leftover byte is a short packet: it names the field, the bytes needed, and the bytes left, and returns no row. A group that reads no bytes at all ends the repeat, and the bytes left are trailing bytes.
|
|
902
|
+
|
|
903
|
+
Two points, `(10, 20)` then `(30, 40)`:
|
|
904
|
+
|
|
905
|
+
```python
|
|
906
|
+
row = Scheme(
|
|
907
|
+
1,
|
|
908
|
+
dict,
|
|
909
|
+
repeat(0, i32(0, lambda row: row["lat"]), i32(1, lambda row: row["lon"])),
|
|
910
|
+
)
|
|
911
|
+
BinaryPacker.pack(row, {"lat": [10, 30], "lon": [20, 40]})
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
```
|
|
915
|
+
01 0a 00 00 00 14 00 00 00 1e 00 00 00 28 00 00 00
|
|
916
|
+
```
|
|
917
|
+
|
|
918
|
+
A field after `repeat` is eaten as another point. Use `times` when something follows the group. C#, TypeScript, Java and Rust refuse a packet with more rounds than the scheme's `maxRounds` (65,535 by default); see [Untrusted input](#untrusted-input).
|
|
919
|
+
|
|
920
|
+
A round can hold more than plain fields: a `when`, `flags` or a group. Give every name the round holds one list. Entry `i` is the value for round `i`, and `repeat` runs as many rounds as its longest list. `repeat(k, when(k == 1, v))` with `k = [1, 2]` and `v = [9]` packs `01 01 09 02`. Unpack gives the same shape: every name a round can hold has one entry per round, and a round that skipped the name has `null` (`undefined` in TypeScript), so a row packs again to the same bytes. TypeScript, C# and Java do this. Python lists only the rounds that read a name. Rust and C++ keep each round as a row: a `Vec<E>` for a typed Rust `times`, `Value::Groups` in the map form of Rust (see Times under Rust), and an `Array<E, N>` in C++. TypeScript rows are flat, so give a group's members as lists under their own names: the nested form `{mark: {v: [7, undefined, 9]}}` throws `missing v`.
|
|
921
|
+
|
|
922
|
+
A `repeat` or `times` cannot sit inside a `repeat` or `times` round, directly or under a `when`, `flags` or group. TypeScript, C# and Java refuse the scheme when you build it. Rust refuses all of these but a `times` inside a `repeat`. Python does not check yet. A `list` or `dict` element starts outside any round.
|
|
923
|
+
|
|
924
|
+
C# packs such a row from a dictionary of lists (`BinaryPacker.Pack(scheme, IReadOnlyDictionary<string, object?>)`). It has no public call that reads one yet: a typed `Unpack` of a packet that holds a `repeat` or `times` round throws `InvalidCastException`.
|
|
925
|
+
|
|
926
|
+
### Sized
|
|
927
|
+
|
|
928
|
+
Raw bytes whose length is an earlier integer. No length of its own. The list must be that many bytes. Count 3 and `75 61 76`:
|
|
929
|
+
|
|
930
|
+
```python
|
|
931
|
+
row = Scheme(
|
|
932
|
+
1,
|
|
933
|
+
dict,
|
|
934
|
+
u16(0, lambda row: row["n"]),
|
|
935
|
+
sized(1, lambda row: row["payload"], 0),
|
|
936
|
+
)
|
|
937
|
+
BinaryPacker.pack(row, {"n": 3, "payload": b"uav"})
|
|
938
|
+
```
|
|
939
|
+
|
|
940
|
+
```
|
|
941
|
+
01 03 00 75 61 76
|
|
942
|
+
```
|
|
943
|
+
|
|
944
|
+
A short buffer names the field, the bytes needed, and the bytes left, and returns no row.
|
|
945
|
+
|
|
946
|
+
### Two-bit slots
|
|
947
|
+
|
|
948
|
+
`u2` packs a fixed list of named slots, four values per byte, low bits first. Each value is 0 … 3. Slots `0, 1, 2, 3` are the single byte `e4`.
|
|
949
|
+
|
|
950
|
+
```python
|
|
951
|
+
row = Scheme(
|
|
952
|
+
1,
|
|
953
|
+
dict,
|
|
954
|
+
u2(
|
|
955
|
+
(0, lambda row: row["a"]),
|
|
956
|
+
(1, lambda row: row["b"]),
|
|
957
|
+
(2, lambda row: row["c"]),
|
|
958
|
+
(3, lambda row: row["d"]),
|
|
959
|
+
),
|
|
960
|
+
)
|
|
961
|
+
BinaryPacker.pack(row, {"a": 0, "b": 1, "c": 2, "d": 3})
|
|
962
|
+
```
|
|
963
|
+
|
|
964
|
+
```
|
|
965
|
+
01 e4
|
|
966
|
+
```
|
|
967
|
+
|
|
968
|
+
One slot whose value is 1 is `01 01`. Unused bits in the last byte are 0.
|
|
969
|
+
|
|
970
|
+
### Bits
|
|
971
|
+
|
|
972
|
+
A list of 0 and 1. The item count is an earlier integer, as stored. Eight 1-bits are `ff`. Nine spill into a second byte whose low bit is 1 and whose other bits are 0.
|
|
973
|
+
|
|
974
|
+
```python
|
|
975
|
+
row = Scheme(1, dict, u8(0, lambda row: row["n"]), bits(1, lambda row: row["segs"], 0))
|
|
976
|
+
BinaryPacker.pack(row, {"n": 8, "segs": [1, 1, 1, 1, 1, 1, 1, 1]})
|
|
977
|
+
```
|
|
978
|
+
|
|
979
|
+
```
|
|
980
|
+
01 08 ff
|
|
981
|
+
```
|
|
982
|
+
|
|
983
|
+
The list length must equal that count. `bits` cannot mean "one less than the count". That is `packed`.
|
|
984
|
+
|
|
985
|
+
### Packed
|
|
986
|
+
|
|
987
|
+
A list of small integers, width 1 or 2, with no length of its own. The item count is an earlier integer plus a bias of 0 or −1. Width 2 stores 0 … 3, four per byte, low bits first. Width 1 stores 0 or 1, eight per byte. Values `0, 1, 2, 3` are `e4`.
|
|
988
|
+
|
|
989
|
+
```python
|
|
990
|
+
kinds = Scheme(
|
|
991
|
+
1,
|
|
992
|
+
dict,
|
|
993
|
+
u8(0, lambda row: row["n"]),
|
|
994
|
+
packed(2, 1, lambda row: row["kinds"], 0),
|
|
995
|
+
)
|
|
996
|
+
BinaryPacker.pack(kinds, {"n": 4, "kinds": [0, 1, 2, 3]})
|
|
997
|
+
```
|
|
998
|
+
|
|
999
|
+
```
|
|
1000
|
+
01 04 e4
|
|
1001
|
+
```
|
|
1002
|
+
|
|
1003
|
+
Bias −1 is the count minus one. A count of 9 and eight 1-bits is `ff`. A count of 1 writes no bitset bytes.
|
|
1004
|
+
|
|
1005
|
+
```python
|
|
1006
|
+
legs = Scheme(
|
|
1007
|
+
1,
|
|
1008
|
+
dict,
|
|
1009
|
+
u8(0, lambda row: row["n"]),
|
|
1010
|
+
packed(1, 1, lambda row: row["legs"], 0, -1),
|
|
1011
|
+
)
|
|
1012
|
+
BinaryPacker.pack(legs, {"n": 9, "legs": [1, 1, 1, 1, 1, 1, 1, 1]})
|
|
1013
|
+
```
|
|
1014
|
+
|
|
1015
|
+
```
|
|
1016
|
+
01 09 ff
|
|
1017
|
+
```
|
|
1018
|
+
|
|
1019
|
+
A list whose length is not that count is rejected, and the error names the field.
|
|
1020
|
+
|
|
1021
|
+
### Times
|
|
1022
|
+
|
|
1023
|
+
The inner fields, exactly N times. N is an earlier integer. The next scheme field is then read as itself. `repeat` would keep going until the buffer ends.
|
|
1024
|
+
|
|
1025
|
+
Count 2, points `(10, 20)` and `(30, 40)`, then a trailing `u8` of 7:
|
|
1026
|
+
|
|
1027
|
+
```python
|
|
1028
|
+
row = Scheme(
|
|
1029
|
+
1,
|
|
1030
|
+
dict,
|
|
1031
|
+
u8(0, lambda row: row["n"]),
|
|
1032
|
+
times(1, 0, i32(1, lambda row: row["lat"]), i32(2, lambda row: row["lon"])),
|
|
1033
|
+
u8(3, lambda row: row["tail"]),
|
|
1034
|
+
)
|
|
1035
|
+
BinaryPacker.pack(row, {"n": 2, "lat": [10, 30], "lon": [20, 40], "tail": 7})
|
|
1036
|
+
```
|
|
1037
|
+
|
|
1038
|
+
```
|
|
1039
|
+
01 02 0a 00 00 00 14 00 00 00 1e 00 00 00 28 00 00 00 07
|
|
1040
|
+
```
|
|
1041
|
+
|
|
1042
|
+
The rules of Repeat hold for the rounds of `times`: one list per name, entry `i` for round `i`, `null` (`undefined` in TypeScript) for a round that skipped the name, and no `repeat` or `times` inside a round. The count is the number of rounds. C# refuses a list longer than the count (`'V' holds 2 items, but the times count is 1`) and a list too short for a round that walks the field. Rust binds a `Vec<E>`; see Times under Rust. The round limits of Repeat apply to the rounds of `times` as well (a count above `maxRounds` is refused when the round past the limit would start); see [Untrusted input](#untrusted-input).
|
|
1043
|
+
|
|
1044
|
+
`packed` and `times` together are how one scheme holds a route: a header, N two-bit point kinds, N latitude/longitude pairs, then N−1 straight-leg bits only when a flag is set. That packet is
|
|
1045
|
+
|
|
1046
|
+
```
|
|
1047
|
+
34 10 00 15 00 06 2d 00 02 0d 00 65 cd 1d 00 a3 e1 11 10 8c cd 1d 10 ca e1 11 01
|
|
1048
|
+
```
|
|
1049
|
+
|
|
1050
|
+
[`_docs/01_solution/schema.md`](_docs/01_solution/schema.md)
|
|
1051
|
+
|
|
1052
|
+
## Untrusted input
|
|
1053
|
+
|
|
1054
|
+
Unpack reads bytes from the network, so a packet may be built to hurt. In C#, TypeScript, Python, Rust and Java, `unpack` answers every packet it cannot read with an error value and no row. It does not throw, except the typed C# `Unpack` of a packet that holds a `repeat` or `times` round (see Limits to keep in mind). Its time grows with the length of the packet. In C#, TypeScript, Java and Rust the memory the rounds of a `repeat` or `times` can take is bounded by the scheme's round limits, not by the packet: a packet with more rounds than the limits allow is refused (see Limits to keep in mind). That is the only amplification the limits remove: a `bits` or `packed` count is bounded only by the bytes left, so those items still cost memory in a straight line with the length of the packet (a 1 MiB packet of 8,388,608 `bits` items measured about 38 MB in C# and Java, about 230 MB in TypeScript and about 270 MB in Rust, net of the runtime), and the limits are per call, so concurrent calls add up. Cap the packet length where you read it from the network. C++ and Python need no round limit; their memory stays small for the reasons given there. The cases it refuses:
|
|
1055
|
+
|
|
1056
|
+
- a count that is negative, or larger than the bytes left
|
|
1057
|
+
- a count whose field is absent because its flag bit was clear
|
|
1058
|
+
- invalid UTF-8 in a string or a dictionary key
|
|
1059
|
+
- a `times` round, or a `list` or `dict` element, that reads no bytes. A `repeat` round that reads none ends the repeat, and the bytes left are trailing bytes
|
|
1060
|
+
|
|
1061
|
+
The kind of error for these is not settled yet. Today they come back as a short-packet-style error, so test for failure, not for its exact kind. The C++ package already reports every failure as a `Result`; it does not check that a string is valid UTF-8.
|
|
1062
|
+
|
|
1063
|
+
These mistakes in a scheme are refused when you build it, not when a packet arrives:
|
|
1064
|
+
|
|
1065
|
+
- A split-form flag bit must come after its flag byte, in the same place: the top level, one `repeat` or `times` round, or one `list` or `dict` element. A flag byte read inside a `when` is not visible after it. TypeScript, C#, Java and Rust refuse a violation at construction; Python does not check yet.
|
|
1066
|
+
- A `when` or a count may name only a field read earlier in its own place: the top level, one `repeat` or `times` round, one `list` or `dict` element, or (C#, TypeScript, Java) one nested row. A field inside a `repeat` or `times` is not visible after it, and an outer field is not visible inside it. C#, TypeScript, Java and C++ refuse a violation at construction. Rust does too for a numeric id that names a later field or an outer field from inside a round; it does not check yet a field inside an earlier `repeat` or `times` body, or a name the scheme never declares. Python does not check yet. TypeScript refuses a count that names a `bool` or a float. In Java the named field of a `when` or a count must be an integer or a `bool`.
|
|
1067
|
+
- A `bool` stands only directly under `flags` or a flag-byte bit, and one flags byte holds at most 8 bits. An empty group follows the same rule when something carries its `true` (its own value in TypeScript and Rust, a `bool` member in C# and C++, a nested-row accessor in Java, where the nested row carries presence only: `{g: {x: 1}}` comes back as `{g: {}}`). An empty group that could never set its bit (Python and Java `group(anchor)` with no fields, a C++ `group(id)` with no fields and no member, a C# empty group bound to a non-`bool` member) fails construction wherever it stands. Every package refuses a violation at construction; in TypeScript `new Scheme(...)` checks the same as `scheme(...)`.
|
|
1068
|
+
- A `repeat` or `times` inside a `repeat` or `times` round, directly or under a `when`, `flags` or group. C#, TypeScript and Java refuse it at construction (`repeat 1 is inside a repeat or times round; a round cannot hold another repeat or times`). Rust refuses all of these but a `times` inside a `repeat`. Python does not check yet. A `list` or `dict` element starts outside any round.
|
|
1069
|
+
- In TypeScript, a member name that group flattening would overwrite: a name declared inside an unanchored group and also outside it or in another unanchored group, or a group named like one of its own members (`member sid: declared inside a group and outside it; a group's members are flattened into the row, so one would overwrite the other`).
|
|
1070
|
+
- In C#, a field declared for another row type, such as `Field.U16<Other>` in a `Scheme<Row>`: `'Q' is declared for row type Other, but this scheme is for R`. A field of a nested row goes inside its `Group`, not in the parent scheme.
|
|
1071
|
+
- In Rust, a `when` on a field that is not an integer or a `bool`, or compared with a value that is not an integer; and a `list` or `dict` element that is a group, `flags`, `when`, `repeat`, `times`, `sized`, `bits`, `packed` or a `u2` with more than one name, which could not round-trip.
|
|
1072
|
+
|
|
1073
|
+
If you upgrade, three schemes that built before now do not (the first two in TypeScript, C#, Java and Rust): a flag byte in one `when` with its bit in another `when`, a flag byte outside a `list`, `repeat` or `times` with its bit inside it, and (Rust) a reference from inside a `repeat` or `times` to a field outside it. Packet bytes do not change. A `times`, `list` or `dict` element that reads nothing used to unpack as an empty item and is now an error.
|
|
1074
|
+
|
|
1075
|
+
The `bool` rule changes more when you upgrade. A `bool` that is `false` now packs a clear bit (`01 00`) in C#, TypeScript and Rust, as Java, Python and C++ already did, so mixed versions read that row differently until both sides upgrade. A `bool` or empty group outside `flags`, an empty group that could never set its bit, and a ninth flag bit now fail at construction. In C#, packing a flags group whose bit is on throws when one of its values is missing, instead of writing a packet that cannot be read, and a group whose only values are `u2`, `sized`, `bits`, `packed` or a nested group now sets its bit. In C# and TypeScript, a value other than `true` (for example `1`) leaves a bool's bit clear. In TypeScript, `new Scheme(...)` now refuses everything `scheme(...)` refuses (a type number above 255, a wrong field id, a split bit outside its scope, a bool outside flags) instead of building it, and its `fields` come back flattened (`flagByte` / `flagBit` in place of `flags`), as `scheme(...)` already returned them. In Java, a `when` or a count that names a later field, a field outside its round, or a field that is not an integer or a `bool` now fails at construction, a `repeat` or `times` nested inside a `repeat` or `times` round fails at construction, and unpacking a `repeat` or `times` gives every field one list entry per round, `null` where the round skipped it. In Rust, `packbin::pack` and `packbin::unpack` for map schemes are now public, a map bool value other than 0 or 1 or a `__repeat__` value that is not groups fails pack, a `repeat` inside a `repeat` or `times` round and a `times` inside a `times` round fail at construction (a `times` inside a `repeat` round still works), a `FlagByte` handle no longer counts bits across schemes, and a second read of a flag byte with the same name starts its own bits, as in C++.
|
|
1076
|
+
|
|
1077
|
+
Reference scope is now checked in C# and TypeScript too, as it already was in Java, Rust (numeric ids) and C++. If you upgrade, a `when` or a count that names a later field, an id that does not exist, or a field of another scope throws at construction: `ArgumentException` in C# (`when 0 names field id 1, which is not an earlier field in the same scope ...`), `RangeError` in TypeScript (`when 0: eq names field id 1, which is not declared earlier in the same scope`). C# used to resolve a later id and pack a packet its own unpack rejected. TypeScript used to search the whole scheme for the id, so a `when` inside an unanchored group could match a field outside it that had the same id; it now reads its own scope. A `sized`, `bits` or `packed` counted by a field of the same `repeat` round now unpacks in TypeScript (it was an error). Python still builds these schemes. A `repeat` or `times` inside a `repeat` or `times` round now fails at construction in C# and TypeScript, as in Java.
|
|
1078
|
+
|
|
1079
|
+
TypeScript pack throws where it used to write wrong bytes. A `RangeError` names the member for an integer out of range (`n: 300 does not fit in u8`; it wrote `2c`), a fraction or `NaN` in an integer field (1.5 was cut to 1), a value that is not a number (`n: expected a number for u8, got string`), a `bigint` or a numeric string in a float field, a finite value too large for an `f32` (`x: 1e+39 does not fit in f32`; it wrote an infinity), and a 64-bit `number` that is not a safe integer (`n: 9007199254740992 does not fit in u64; pass a bigint`). A field under `flags` inside a group is now written: a group of `u8 a` and `flags(u8 c, u8 d)` with `{g: {a: 7, c: 2}}` packed `01 07 00` and packs `01 07 01 02`. A member name that group flattening would overwrite fails at construction. These checks run in `new Scheme(...)` too. `eq` compares numbers and bigints by value, so a `when` on a `u64` field matches on pack and on unpack. `Scheme.fields` holds copies of the `when`, `sized`, `bits`, `packed` and `times` fields with the name they refer to filled in, not the objects you passed in.
|
|
1080
|
+
|
|
1081
|
+
TypeScript unpacked rows changed. The flag-byte value is no longer a member (`flags` used to add a `""` key, and `flagByte("m")` added `m`). In a `repeat` or `times` round every name the round can hold is a list with one entry per round, `undefined` where the round skipped it (a `bool` under `flags` is `true` or `undefined`); the list used to hold only the rounds that read the name. Pack reads these lists by round index, so a row packs again to the same bytes, and a `when`, `flags` or group inside a round packs. `repeat(k, when(k == 1, v))` with `{k: [1, 2], v: [9]}` packs `01 01 09 02`; it threw `expected number`.
|
|
1082
|
+
|
|
1083
|
+
C# pack no longer drops data. `Pack` throws `ArgumentException` where a value is missing outside a flag bit (`'N' (field id 1) has no value; a field that may be absent belongs in Flags`). It used to write a shorter packet that the peer misread, or throw `KeyNotFoundException` for an absent list, dictionary or `Sized`. The same error comes when a round runs out of list, and when a `times` list is shorter than its count. A longer one throws `'V' holds 2 items, but the times count is 1`. A count whose field pack did not write (a `when` or a clear flag bit skipped it) throws `InvalidOperationException`: `D: count 'N' is missing`. A scheme that holds a field declared for another row type throws `ArgumentException` at construction: `Field.U16<Other>` in a `Scheme<Row>` used to leave its value out, and a child row's field used directly in the parent scheme, which packed, now goes inside its `Group`. Pack decides every `when` from what it wrote, as unpack does from what it read. A `when` on a field that an earlier `when` or a clear flag bit skipped no longer matches (a row that packed `01 01 04` now packs `01 01`), and `Eq(bool, false)` on a clear bit no longer matches. A float `when` follows Java's `Double.compare`: NaN equals NaN and -0.0 differs from 0.0. Give an `f32` field a `float` in `Eq` (`Eq(0, 0.1f)`): `Eq(0, 0.1)` matches nothing now. A scheme owns a copy of its fields, so one `Field` or `Condition` can be used in several schemes (a second scheme with the same `Condition` used to rebind the first). A `FlagByte` handle is still shared: add all its bits before you build the scheme. In a round, pack reads values under `when`, `flags` and groups by round index, and a read gives aligned lists with `null` for a skipped round; there is no public call that returns such a row yet (see Repeat).
|
|
1084
|
+
|
|
1085
|
+
Rust `times` and `when` changed. A typed `times` is now `SchemeItem::times(anchor, count_id, get, set, members)` and binds a `Vec<E>` (see Times under Rust). The three-argument form and the `SchemeItem::Times { .. }` struct variant are removed, so typed code that used them does not compile. A scheme that built before can now fail at construction: a `when` on a float, utf8 or bytes field or compared with a value that is not an integer (`when at id 1 tests field "0", which is not an integer or bool`), and a `list` or `dict` element that is a group, `flags`, `when`, `repeat`, `times`, `sized`, `bits`, `packed` or a `u2` with more than one name. A `when` now compares by integer value at any width; `eq` of a `u8` against a `u16` field used to never match. Two bound lists or dicts in one typed scheme now keep their own members; the second used to overwrite the first, and pack wrote it twice. The map form of `times` also returns its rounds under `__times_<anchor>`, and pack refuses a list that disagrees with them: `times at id 1: list for '1' disagrees with its rounds`. Edit the rounds and drop or rewrite the per-name lists, or edit the lists and remove the `__times_<anchor>` key.
|
|
1086
|
+
|
|
1087
|
+
Round limits change what unpack accepts. If you upgrade C#, TypeScript, Java or Rust, a packet with more than 65,535 rounds in one `repeat` or `times` field, or whose rounds together hold more than 4,194,304 slots, that unpacked before is now refused until its scheme raises the limit; packet bytes and every result for a smaller packet do not change. In C# and TypeScript, an omitted argument of `WithLimits` or `withLimits` is the default, not the scheme's current value, so `scheme.WithLimits(maxRounds: 200_000).WithLimits(maxSlots: 16_000_000)` ends with the default `maxRounds` again: give both in one call. Java and Rust take both values every time.
|
|
1088
|
+
|
|
1089
|
+
Limits to keep in mind:
|
|
1090
|
+
|
|
1091
|
+
- Round limits (C#, TypeScript, Java and Rust). A scheme carries two limits on one unpack call. `maxRounds`, default 65,535, is the most rounds one `repeat` or `times` field may start. `maxSlots`, default 4,194,304, is the most slots all the rounds of the call may create together. A slot is one entry of a round's lists, one for each name the round can hold, even where the round read nothing, so a round of a 36-name `when` body takes 36 slots. The packages count the names a little differently (C# distinct member names, TypeScript the names of the round, Java its value fields, Rust its fields), so a packet right at the slot limit can pass in one package and fail in another. A packet that would pass either limit is refused when the round that crosses it starts, before the round's bytes are read and before anything is allocated for it, with the same bad-value error as any other unreadable value (see above): no row, handler not called. A `times` count above `maxRounds` is not refused up front; a packet that states a huge count and holds few rounds still ends in a short read. Raise or lower the limits where you declare the scheme: `scheme.WithLimits(maxRounds: 200_000, maxSlots: 16_000_000)` (C#, `Scheme<T>`), `scheme.withLimits({ maxRounds: 200_000, maxSlots: 16_000_000 })` (TypeScript), `scheme.withLimits(200_000, 16_000_000L)` (Java) and `scheme.with_limits(200_000, 16_000_000)` (Rust, on `MapScheme` and the typed `Scheme`; it takes and returns the scheme). The defaults are constants (`BinaryPacker.DefaultMaxRounds` and `DefaultMaxSlots`, `Scheme.DefaultMaxRounds` and `DefaultMaxSlots`, `Scheme.DEFAULT_MAX_ROUNDS` and `DEFAULT_MAX_SLOTS`, `packbin::DEFAULT_MAX_ROUNDS` and `DEFAULT_MAX_SLOTS`). A value below 1 is refused when you call it (an exception in C#, TypeScript and Java; a panic in Rust). C++ and Python have no limit and need none: C++ unpacks into `Array<T, N>` storage you own, of capacity at most 65,535, and returns `Error::TooMany` for a longer count, and Python keeps only the values it reads (an unread name costs nothing), so its memory does not grow with the names of a round (about 36 MiB for the packet above, measured with Python 3.14). Time still grows with the length of the packet, so cap the packet length where you read it from the network as well.
|
|
1092
|
+
- Memory at the defaults, for a 1 MiB packet of one-byte rounds in a `repeat` with a 36-name `when` body (resident set, runtime included; measured on a macOS arm64 laptop with Node 22, .NET 10, Java 21 and Rust 1.79 in release mode, three runs each; your figures will differ). Refused at round 65,536: about 86 MiB in TypeScript, 86 MiB in C# and 92 MiB in Java, where an unlimited run took about 400, 530 and 560 MiB. A Rust one-byte round cannot hold 36 names, so Rust is measured on a `times` of one-byte rounds, about 21 MiB (about 310 MiB unlimited), and on rounds that each set eight flag bits, about 64 MiB (about 1.3 to 1.4 GiB unlimited). A packet that stays inside the limits costs up to the limits and no more: 65,535 rounds of the 36-name body, accepted, peaked at about 106 MiB in TypeScript and 86 MiB in Java, and a 64-name body stopped by the slot limit at about 110 MiB in Java and C#. The C# packet that took 1.4 seconds to read is now refused in about 0.1 seconds. The ceiling is higher when the rounds set their names: a 300-name body stopped by the slot limit peaked at about 190 MB in C#, 350 MB in Java and 130 MB in TypeScript (net of the runtime, measured by the security audit), and 64 set flag bits per round in Rust peaked at about 430 MB; raise the limits only as far as your memory per concurrent call allows.
|
|
1093
|
+
- In C#, a typed `Unpack` of a packet that holds a `repeat` or `times` round throws `InvalidCastException` instead of returning an error value: the row type has no member that can hold a list per round, and no public C# call returns one yet.
|
|
1094
|
+
- In TypeScript a decoded dictionary keeps a key named `__proto__` as an ordinary own entry. Copy or merge decoded dictionaries with care: `Object.assign({}, dict)` sets the target's prototype from that entry. Read them with `Object.hasOwn`.
|
|
1095
|
+
- In C# a `u64` or `i64` field arrives in `UnpackResult.Values` as `ulong` or `long`. Read through the row type and nothing changes for you; code that reads those two kinds straight from `Values` as `double` must change.
|
|
1096
|
+
- A session packs from several threads safely in C# and Java (each packet gets its own number). Unpacking on one session must be called in packet order, one caller at a time.
|
|
1097
|
+
|
|
1098
|
+
## License
|
|
1099
|
+
|
|
1100
|
+
MIT
|