tigerbeetle-node 0.11.9 → 0.11.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.client.node.sha256 +1 -1
- package/dist/index.d.ts +70 -67
- package/dist/index.js +70 -67
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/scripts/build_lib.sh +22 -10
- package/src/index.ts +3 -0
- package/src/tigerbeetle/src/config.zig +2 -0
- package/src/tigerbeetle/src/constants.zig +18 -0
- package/src/tigerbeetle/src/lsm/forest_fuzz.zig +0 -2
- package/src/tigerbeetle/src/lsm/manifest_log_fuzz.zig +6 -5
- package/src/tigerbeetle/src/lsm/test.zig +5 -4
- package/src/tigerbeetle/src/lsm/tree.zig +3 -30
- package/src/tigerbeetle/src/lsm/tree_fuzz.zig +0 -2
- package/src/tigerbeetle/src/simulator.zig +34 -53
- package/src/tigerbeetle/src/testing/cluster/state_checker.zig +1 -1
- package/src/tigerbeetle/src/testing/cluster/storage_checker.zig +6 -6
- package/src/tigerbeetle/src/testing/cluster.zig +5 -99
- package/src/tigerbeetle/src/testing/storage.zig +26 -23
- package/src/tigerbeetle/src/vsr/journal.zig +86 -90
- package/src/tigerbeetle/src/vsr/replica.zig +631 -687
- package/src/tigerbeetle/src/vsr/replica_format.zig +12 -12
- package/src/tigerbeetle/src/vsr/superblock.zig +192 -110
- package/src/tigerbeetle/src/vsr/superblock_fuzz.zig +35 -8
- package/src/tigerbeetle/src/vsr/superblock_quorums.zig +48 -48
- package/src/tigerbeetle/src/vsr/superblock_quorums_fuzz.zig +50 -50
- package/src/tigerbeetle/src/vsr.zig +93 -29
|
@@ -22,7 +22,7 @@ const StorageFaultAtlas = @import("../testing/storage.zig").ClusterFaultAtlas;
|
|
|
22
22
|
const MessagePool = @import("../message_pool.zig").MessagePool;
|
|
23
23
|
const superblock_zone_size = @import("superblock.zig").superblock_zone_size;
|
|
24
24
|
const data_file_size_min = @import("superblock.zig").data_file_size_min;
|
|
25
|
-
const VSRState = @import("superblock.zig").
|
|
25
|
+
const VSRState = @import("superblock.zig").SuperBlockHeader.VSRState;
|
|
26
26
|
const SuperBlockType = @import("superblock.zig").SuperBlockType;
|
|
27
27
|
const SuperBlock = SuperBlockType(Storage);
|
|
28
28
|
const fuzz = @import("../testing/fuzz.zig");
|
|
@@ -151,9 +151,10 @@ const Environment = struct {
|
|
|
151
151
|
/// Indexed by sequence.
|
|
152
152
|
const SequenceStates = std.ArrayList(struct {
|
|
153
153
|
vsr_state: VSRState,
|
|
154
|
+
vsr_headers: vsr.ViewChangeHeaders.BoundedArray,
|
|
154
155
|
/// Track the expected `checksum(free_set)`.
|
|
155
156
|
/// Note that this is a checksum of the decoded free set; it is not the same as
|
|
156
|
-
/// `
|
|
157
|
+
/// `SuperBlockHeader.free_set_checksum`.
|
|
157
158
|
free_set: u128,
|
|
158
159
|
});
|
|
159
160
|
|
|
@@ -166,7 +167,7 @@ const Environment = struct {
|
|
|
166
167
|
latest_sequence: u64 = 0,
|
|
167
168
|
latest_checksum: u128 = 0,
|
|
168
169
|
latest_parent: u128 = 0,
|
|
169
|
-
latest_vsr_state: VSRState = std.mem.
|
|
170
|
+
latest_vsr_state: VSRState = std.mem.zeroes(VSRState),
|
|
170
171
|
|
|
171
172
|
context_format: SuperBlock.Context = undefined,
|
|
172
173
|
context_open: SuperBlock.Context = undefined,
|
|
@@ -232,7 +233,7 @@ const Environment = struct {
|
|
|
232
233
|
if (env.latest_checksum != 0) {
|
|
233
234
|
if (env.latest_sequence + 1 == env.superblock_verify.working.sequence) {
|
|
234
235
|
// After a checkpoint() or view_change(), the parent points to the previous
|
|
235
|
-
// working
|
|
236
|
+
// working header.
|
|
236
237
|
assert(env.superblock_verify.working.parent == env.latest_checksum);
|
|
237
238
|
}
|
|
238
239
|
}
|
|
@@ -268,10 +269,14 @@ const Environment = struct {
|
|
|
268
269
|
.replica = 0,
|
|
269
270
|
});
|
|
270
271
|
|
|
272
|
+
var vsr_headers = vsr.ViewChangeHeaders.BoundedArray{ .buffer = undefined };
|
|
273
|
+
vsr_headers.appendAssumeCapacity(vsr.Header.root_prepare(cluster));
|
|
274
|
+
|
|
271
275
|
assert(env.sequence_states.items.len == 0);
|
|
272
276
|
try env.sequence_states.append(undefined); // skip sequence=0
|
|
273
277
|
try env.sequence_states.append(.{
|
|
274
278
|
.vsr_state = VSRState.root(cluster),
|
|
279
|
+
.vsr_headers = vsr_headers,
|
|
275
280
|
.free_set = checksum_free_set(env.superblock),
|
|
276
281
|
});
|
|
277
282
|
}
|
|
@@ -310,14 +315,29 @@ const Environment = struct {
|
|
|
310
315
|
.view = env.superblock.staging.vsr_state.view + 5,
|
|
311
316
|
};
|
|
312
317
|
|
|
318
|
+
var vsr_headers = vsr.ViewChangeHeaders.BoundedArray{ .buffer = undefined };
|
|
319
|
+
var vsr_head = std.mem.zeroInit(vsr.Header, .{
|
|
320
|
+
.command = .prepare,
|
|
321
|
+
.op = env.superblock.staging.vsr_state.commit_min,
|
|
322
|
+
});
|
|
323
|
+
vsr_head.set_checksum_body(&.{});
|
|
324
|
+
vsr_head.set_checksum();
|
|
325
|
+
vsr_headers.appendAssumeCapacity(vsr_head);
|
|
326
|
+
|
|
313
327
|
assert(env.sequence_states.items.len == env.superblock.staging.sequence + 1);
|
|
314
328
|
try env.sequence_states.append(.{
|
|
315
329
|
.vsr_state = vsr_state,
|
|
330
|
+
.vsr_headers = vsr_headers,
|
|
316
331
|
.free_set = env.sequence_states.items[env.sequence_states.items.len - 1].free_set,
|
|
317
332
|
});
|
|
318
333
|
|
|
319
334
|
env.pending.insert(.view_change);
|
|
320
|
-
env.superblock.view_change(view_change_callback, &env.context_view_change,
|
|
335
|
+
env.superblock.view_change(view_change_callback, &env.context_view_change, .{
|
|
336
|
+
.commit_max = vsr_state.commit_max,
|
|
337
|
+
.log_view = vsr_state.log_view,
|
|
338
|
+
.view = vsr_state.view,
|
|
339
|
+
.headers = vsr_headers,
|
|
340
|
+
});
|
|
321
341
|
}
|
|
322
342
|
|
|
323
343
|
fn view_change_callback(context: *SuperBlock.Context) void {
|
|
@@ -334,18 +354,25 @@ const Environment = struct {
|
|
|
334
354
|
.commit_min_checksum = env.superblock.staging.vsr_state.commit_min_checksum + 1,
|
|
335
355
|
.commit_min = env.superblock.staging.vsr_state.commit_min + 1,
|
|
336
356
|
.commit_max = env.superblock.staging.vsr_state.commit_max + 1,
|
|
337
|
-
.log_view = env.superblock.staging.vsr_state.log_view
|
|
338
|
-
.view = env.superblock.staging.vsr_state.view
|
|
357
|
+
.log_view = env.superblock.staging.vsr_state.log_view,
|
|
358
|
+
.view = env.superblock.staging.vsr_state.view,
|
|
339
359
|
};
|
|
340
360
|
|
|
341
361
|
assert(env.sequence_states.items.len == env.superblock.staging.sequence + 1);
|
|
342
362
|
try env.sequence_states.append(.{
|
|
343
363
|
.vsr_state = vsr_state,
|
|
364
|
+
.vsr_headers = vsr.ViewChangeHeaders.BoundedArray.fromSlice(
|
|
365
|
+
env.superblock.staging.vsr_headers().slice,
|
|
366
|
+
) catch unreachable,
|
|
344
367
|
.free_set = checksum_free_set(env.superblock),
|
|
345
368
|
});
|
|
346
369
|
|
|
347
370
|
env.pending.insert(.checkpoint);
|
|
348
|
-
env.superblock.checkpoint(checkpoint_callback, &env.context_checkpoint,
|
|
371
|
+
env.superblock.checkpoint(checkpoint_callback, &env.context_checkpoint, .{
|
|
372
|
+
.commit_min_checksum = vsr_state.commit_min_checksum,
|
|
373
|
+
.commit_min = vsr_state.commit_min,
|
|
374
|
+
.commit_max = vsr_state.commit_max,
|
|
375
|
+
});
|
|
349
376
|
}
|
|
350
377
|
|
|
351
378
|
fn checkpoint_callback(context: *SuperBlock.Context) void {
|
|
@@ -3,7 +3,7 @@ const assert = std.debug.assert;
|
|
|
3
3
|
const log = std.log.scoped(.superblock_quorums);
|
|
4
4
|
|
|
5
5
|
const superblock = @import("./superblock.zig");
|
|
6
|
-
const
|
|
6
|
+
const SuperBlockHeader = superblock.SuperBlockHeader;
|
|
7
7
|
const SuperBlockVersion = superblock.SuperBlockVersion;
|
|
8
8
|
const fuzz = @import("./superblock_quorums_fuzz.zig");
|
|
9
9
|
|
|
@@ -16,10 +16,10 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
16
16
|
const Quorums = @This();
|
|
17
17
|
|
|
18
18
|
const Quorum = struct {
|
|
19
|
-
|
|
19
|
+
header: *const SuperBlockHeader,
|
|
20
20
|
valid: bool = false,
|
|
21
21
|
/// Track which copies are a member of the quorum.
|
|
22
|
-
/// Used to ignore duplicate copies of a
|
|
22
|
+
/// Used to ignore duplicate copies of a header when determining a quorum.
|
|
23
23
|
copies: QuorumCount = QuorumCount.initEmpty(),
|
|
24
24
|
/// An integer value indicates the copy index found in the corresponding slot.
|
|
25
25
|
/// A `null` value indicates that the copy is invalid or not a member of the working
|
|
@@ -86,7 +86,7 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
86
86
|
/// trailers may be damaged.
|
|
87
87
|
pub fn working(
|
|
88
88
|
quorums: *Quorums,
|
|
89
|
-
copies: []
|
|
89
|
+
copies: []SuperBlockHeader,
|
|
90
90
|
threshold: Threshold,
|
|
91
91
|
) Error!Quorum {
|
|
92
92
|
assert(copies.len == options.superblock_copies);
|
|
@@ -102,17 +102,17 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
102
102
|
for (quorums.slice()) |quorum| {
|
|
103
103
|
if (quorum.copies.count() == options.superblock_copies) {
|
|
104
104
|
log.debug("quorum: checksum={x} parent={x} sequence={} count={} valid={}", .{
|
|
105
|
-
quorum.
|
|
106
|
-
quorum.
|
|
107
|
-
quorum.
|
|
105
|
+
quorum.header.checksum,
|
|
106
|
+
quorum.header.parent,
|
|
107
|
+
quorum.header.sequence,
|
|
108
108
|
quorum.copies.count(),
|
|
109
109
|
quorum.valid,
|
|
110
110
|
});
|
|
111
111
|
} else {
|
|
112
112
|
log.warn("quorum: checksum={x} parent={x} sequence={} count={} valid={}", .{
|
|
113
|
-
quorum.
|
|
114
|
-
quorum.
|
|
115
|
-
quorum.
|
|
113
|
+
quorum.header.checksum,
|
|
114
|
+
quorum.header.parent,
|
|
115
|
+
quorum.header.sequence,
|
|
116
116
|
quorum.copies.count(),
|
|
117
117
|
quorum.valid,
|
|
118
118
|
});
|
|
@@ -128,7 +128,7 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
128
128
|
// Verify that the remaining quorums are correctly sorted:
|
|
129
129
|
for (quorums.slice()[1..]) |a| {
|
|
130
130
|
assert(sort_priority_descending({}, b, a));
|
|
131
|
-
assert(a.
|
|
131
|
+
assert(a.header.valid_checksum());
|
|
132
132
|
}
|
|
133
133
|
|
|
134
134
|
// Even the best copy with the most quorum still has inadequate quorum.
|
|
@@ -140,61 +140,61 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
140
140
|
// - a lost or misdirected write
|
|
141
141
|
// - a latent sector error that prevented a write
|
|
142
142
|
for (quorums.slice()[1..]) |a| {
|
|
143
|
-
if (a.
|
|
143
|
+
if (a.header.cluster != b.header.cluster) {
|
|
144
144
|
log.warn("superblock copy={} has cluster={} instead of {}", .{
|
|
145
|
-
a.
|
|
146
|
-
a.
|
|
147
|
-
b.
|
|
145
|
+
a.header.copy,
|
|
146
|
+
a.header.cluster,
|
|
147
|
+
b.header.cluster,
|
|
148
148
|
});
|
|
149
149
|
continue;
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
-
if (a.
|
|
152
|
+
if (a.header.replica != b.header.replica) {
|
|
153
153
|
log.warn("superblock copy={} has replica={} instead of {}", .{
|
|
154
|
-
a.
|
|
155
|
-
a.
|
|
156
|
-
b.
|
|
154
|
+
a.header.copy,
|
|
155
|
+
a.header.replica,
|
|
156
|
+
b.header.replica,
|
|
157
157
|
});
|
|
158
158
|
continue;
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
if (a.
|
|
161
|
+
if (a.header.sequence == b.header.sequence) {
|
|
162
162
|
// Two quorums, same cluster+replica+sequence, but different checksums.
|
|
163
163
|
// This shouldn't ever happen — but if it does, we can't safely repair.
|
|
164
|
-
assert(a.
|
|
164
|
+
assert(a.header.checksum != b.header.checksum);
|
|
165
165
|
return error.Fork;
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
if (a.
|
|
168
|
+
if (a.header.sequence > b.header.sequence + 1) {
|
|
169
169
|
// We read sequences such as (2,2,2,4) — 2 isn't safe to use, but there isn't a
|
|
170
170
|
// valid quorum for 4 either.
|
|
171
171
|
return error.ParentSkipped;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
if (a.
|
|
175
|
-
assert(a.
|
|
176
|
-
assert(a.
|
|
177
|
-
assert(a.
|
|
174
|
+
if (a.header.sequence + 1 == b.header.sequence) {
|
|
175
|
+
assert(a.header.checksum != b.header.checksum);
|
|
176
|
+
assert(a.header.cluster == b.header.cluster);
|
|
177
|
+
assert(a.header.replica == b.header.replica);
|
|
178
178
|
|
|
179
|
-
if (a.
|
|
179
|
+
if (a.header.checksum != b.header.parent) {
|
|
180
180
|
return error.ParentNotConnected;
|
|
181
|
-
} else if (!a.
|
|
181
|
+
} else if (!a.header.vsr_state.monotonic(b.header.vsr_state)) {
|
|
182
182
|
return error.VSRStateNotMonotonic;
|
|
183
183
|
} else {
|
|
184
|
-
assert(b.
|
|
184
|
+
assert(b.header.valid_checksum());
|
|
185
185
|
|
|
186
186
|
return b;
|
|
187
187
|
}
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
assert(b.
|
|
191
|
+
assert(b.header.valid_checksum());
|
|
192
192
|
return b;
|
|
193
193
|
}
|
|
194
194
|
|
|
195
195
|
fn count_copy(
|
|
196
196
|
quorums: *Quorums,
|
|
197
|
-
copy: *const
|
|
197
|
+
copy: *const SuperBlockHeader,
|
|
198
198
|
slot: usize,
|
|
199
199
|
threshold: Threshold,
|
|
200
200
|
) void {
|
|
@@ -240,12 +240,12 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
240
240
|
}
|
|
241
241
|
|
|
242
242
|
var quorum = quorums.find_or_insert_quorum_for_copy(copy);
|
|
243
|
-
assert(quorum.
|
|
244
|
-
assert(quorum.
|
|
243
|
+
assert(quorum.header.checksum == copy.checksum);
|
|
244
|
+
assert(quorum.header.equal(copy));
|
|
245
245
|
|
|
246
246
|
if (copy.copy >= options.superblock_copies) {
|
|
247
|
-
// This
|
|
248
|
-
// The "
|
|
247
|
+
// This header is a valid member of the quorum, but with an unexpected copy number.
|
|
248
|
+
// The "SuperBlockHeader.copy" field is not protected by the checksum, so if that byte
|
|
249
249
|
// (and only that byte) is corrupted, the superblock is still valid — but we don't know
|
|
250
250
|
// for certain which copy this was supposed to be.
|
|
251
251
|
// We make the assumption that this was not a double-fault (corrupt + misdirect) —
|
|
@@ -262,13 +262,13 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
262
262
|
quorum.valid = quorum.copies.count() >= threshold.count();
|
|
263
263
|
}
|
|
264
264
|
|
|
265
|
-
fn find_or_insert_quorum_for_copy(quorums: *Quorums, copy: *const
|
|
265
|
+
fn find_or_insert_quorum_for_copy(quorums: *Quorums, copy: *const SuperBlockHeader) *Quorum {
|
|
266
266
|
assert(copy.valid_checksum());
|
|
267
267
|
|
|
268
268
|
for (quorums.array[0..quorums.count]) |*quorum| {
|
|
269
|
-
if (copy.checksum == quorum.
|
|
269
|
+
if (copy.checksum == quorum.header.checksum) return quorum;
|
|
270
270
|
} else {
|
|
271
|
-
quorums.array[quorums.count] = Quorum{ .
|
|
271
|
+
quorums.array[quorums.count] = Quorum{ .header = copy };
|
|
272
272
|
quorums.count += 1;
|
|
273
273
|
|
|
274
274
|
return &quorums.array[quorums.count - 1];
|
|
@@ -280,26 +280,26 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
280
280
|
}
|
|
281
281
|
|
|
282
282
|
fn sort_priority_descending(_: void, a: Quorum, b: Quorum) bool {
|
|
283
|
-
assert(a.
|
|
283
|
+
assert(a.header.checksum != b.header.checksum);
|
|
284
284
|
|
|
285
285
|
if (a.valid and !b.valid) return true;
|
|
286
286
|
if (b.valid and !a.valid) return false;
|
|
287
287
|
|
|
288
|
-
if (a.
|
|
289
|
-
if (b.
|
|
288
|
+
if (a.header.sequence > b.header.sequence) return true;
|
|
289
|
+
if (b.header.sequence > a.header.sequence) return false;
|
|
290
290
|
|
|
291
291
|
if (a.copies.count() > b.copies.count()) return true;
|
|
292
292
|
if (b.copies.count() > a.copies.count()) return false;
|
|
293
293
|
|
|
294
294
|
// The sort order must be stable and deterministic:
|
|
295
|
-
return a.
|
|
295
|
+
return a.header.checksum > b.header.checksum;
|
|
296
296
|
}
|
|
297
297
|
|
|
298
298
|
/// Repair a quorum's copies in the safest known order.
|
|
299
299
|
/// Repair is complete when every copy is on-disk (not necessarily in its home slot).
|
|
300
300
|
///
|
|
301
|
-
/// We must be careful when repairing superblock
|
|
302
|
-
/// an additional fault occurs. We primarily guard against torn
|
|
301
|
+
/// We must be careful when repairing superblock headers to avoid endangering our quorum if
|
|
302
|
+
/// an additional fault occurs. We primarily guard against torn header writes — preventing a
|
|
303
303
|
/// misdirected write from derailing repair is far more expensive and complex — but they are
|
|
304
304
|
/// likewise far less likely to occur.
|
|
305
305
|
///
|
|
@@ -342,9 +342,9 @@ pub fn QuorumsType(comptime options: Options) type {
|
|
|
342
342
|
}
|
|
343
343
|
|
|
344
344
|
// In descending order, our priorities for repair are:
|
|
345
|
-
// 1. The slot holds no
|
|
346
|
-
// 2. The slot holds no
|
|
347
|
-
// 3. The slot holds a misdirected
|
|
345
|
+
// 1. The slot holds no header, and the copy was not found anywhere.
|
|
346
|
+
// 2. The slot holds no header, but its copy was found elsewhere.
|
|
347
|
+
// 3. The slot holds a misdirected header, but that copy is in another slot as well.
|
|
348
348
|
var a: ?u8 = null;
|
|
349
349
|
var b: ?u8 = null;
|
|
350
350
|
var c: ?u8 = null;
|
|
@@ -388,7 +388,7 @@ test "Quorum.repairs" {
|
|
|
388
388
|
defer std.testing.log_level = level;
|
|
389
389
|
|
|
390
390
|
try fuzz.fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 4 });
|
|
391
|
-
// TODO: Enable these once
|
|
391
|
+
// TODO: Enable these once SuperBlockHeader is generic over its Constants.
|
|
392
392
|
// try fuzz.fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 6 });
|
|
393
393
|
// try fuzz.fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 8 });
|
|
394
394
|
}
|
|
@@ -3,7 +3,7 @@ const assert = std.debug.assert;
|
|
|
3
3
|
const log = std.log.scoped(.fuzz_vsr_superblock_quorums);
|
|
4
4
|
|
|
5
5
|
const superblock = @import("./superblock.zig");
|
|
6
|
-
const
|
|
6
|
+
const SuperBlockHeader = superblock.SuperBlockHeader;
|
|
7
7
|
const SuperBlockVersion = superblock.SuperBlockVersion;
|
|
8
8
|
|
|
9
9
|
const fuzz = @import("../testing/fuzz.zig");
|
|
@@ -21,7 +21,7 @@ pub fn main() !void {
|
|
|
21
21
|
try fuzz_quorums_working(prng.random());
|
|
22
22
|
|
|
23
23
|
try fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 4 });
|
|
24
|
-
// TODO: Enable these once
|
|
24
|
+
// TODO: Enable these once SuperBlockHeader is generic over its Constants.
|
|
25
25
|
// try fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 6 });
|
|
26
26
|
// try fuzz_quorum_repairs(prng.random(), .{ .superblock_copies = 8 });
|
|
27
27
|
}
|
|
@@ -108,18 +108,18 @@ fn test_quorums_working(
|
|
|
108
108
|
const Quorums = QuorumsType(.{ .superblock_copies = 4 });
|
|
109
109
|
const misdirect = random.boolean(); // true:cluster false:replica
|
|
110
110
|
var quorums: Quorums = undefined;
|
|
111
|
-
var
|
|
111
|
+
var headers: [4]SuperBlockHeader = undefined;
|
|
112
112
|
// TODO(Zig): Ideally this would be a [6]?u128 and the access would be
|
|
113
113
|
// "checksums[i] orelse random.int(u128)", but that currently causes the compiler to segfault
|
|
114
114
|
// during code generation.
|
|
115
115
|
var checksums: [6]u128 = undefined;
|
|
116
116
|
for (checksums) |*c| c.* = random.int(u128);
|
|
117
117
|
|
|
118
|
-
// Create
|
|
118
|
+
// Create headers in ascending-sequence order to build the checksum/parent hash chain.
|
|
119
119
|
std.sort.sort(CopyTemplate, copies, {}, CopyTemplate.less_than);
|
|
120
120
|
|
|
121
|
-
for (
|
|
122
|
-
|
|
121
|
+
for (headers) |*header, i| {
|
|
122
|
+
header.* = std.mem.zeroInit(SuperBlockHeader, .{
|
|
123
123
|
.copy = @intCast(u8, i),
|
|
124
124
|
.version = SuperBlockVersion,
|
|
125
125
|
.replica = 1,
|
|
@@ -131,32 +131,32 @@ fn test_quorums_working(
|
|
|
131
131
|
var checksum: ?u128 = null;
|
|
132
132
|
switch (copies[i].variant) {
|
|
133
133
|
.valid => {},
|
|
134
|
-
.valid_high_copy =>
|
|
134
|
+
.valid_high_copy => header.copy = 4,
|
|
135
135
|
.invalid_broken => {
|
|
136
136
|
if (random.boolean() and i > 0) {
|
|
137
|
-
// Error: duplicate
|
|
138
|
-
|
|
137
|
+
// Error: duplicate header (if available).
|
|
138
|
+
header.* = headers[random.uintLessThanBiased(usize, i)];
|
|
139
139
|
checksum = random.int(u128);
|
|
140
140
|
} else {
|
|
141
141
|
// Error: invalid checksum.
|
|
142
142
|
checksum = random.int(u128);
|
|
143
143
|
}
|
|
144
144
|
},
|
|
145
|
-
.invalid_fork =>
|
|
146
|
-
.invalid_parent =>
|
|
145
|
+
.invalid_fork => header.storage_size_max += 1, // Ensure we have a different checksum.
|
|
146
|
+
.invalid_parent => header.parent += 1,
|
|
147
147
|
.invalid_misdirect => {
|
|
148
148
|
if (misdirect) {
|
|
149
|
-
|
|
149
|
+
header.cluster += 1;
|
|
150
150
|
} else {
|
|
151
|
-
|
|
151
|
+
header.replica += 1;
|
|
152
152
|
}
|
|
153
153
|
},
|
|
154
|
-
.invalid_vsr_state =>
|
|
154
|
+
.invalid_vsr_state => header.vsr_state.view += 1,
|
|
155
155
|
}
|
|
156
|
-
|
|
156
|
+
header.checksum = checksum orelse header.calculate_checksum();
|
|
157
157
|
|
|
158
158
|
if (copies[i].variant == .valid or copies[i].variant == .invalid_vsr_state) {
|
|
159
|
-
checksums[
|
|
159
|
+
checksums[header.sequence] = header.checksum;
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
|
|
@@ -165,7 +165,7 @@ fn test_quorums_working(
|
|
|
165
165
|
} else {
|
|
166
166
|
// Shuffling copies can only change the working quorum when we have a corrupt copy index,
|
|
167
167
|
// because we guess that the true index is the slot.
|
|
168
|
-
random.shuffle(
|
|
168
|
+
random.shuffle(SuperBlockHeader, &headers);
|
|
169
169
|
}
|
|
170
170
|
|
|
171
171
|
const threshold = switch (threshold_count) {
|
|
@@ -175,8 +175,8 @@ fn test_quorums_working(
|
|
|
175
175
|
};
|
|
176
176
|
assert(threshold.count() == threshold_count);
|
|
177
177
|
|
|
178
|
-
if (quorums.working(&
|
|
179
|
-
try std.testing.expectEqual(result, working.
|
|
178
|
+
if (quorums.working(&headers, threshold)) |working| {
|
|
179
|
+
try std.testing.expectEqual(result, working.header.sequence);
|
|
180
180
|
} else |err| {
|
|
181
181
|
try std.testing.expectEqual(result, err);
|
|
182
182
|
}
|
|
@@ -238,7 +238,7 @@ pub const CopyTemplate = struct {
|
|
|
238
238
|
}
|
|
239
239
|
};
|
|
240
240
|
|
|
241
|
-
// Verify that a torn
|
|
241
|
+
// Verify that a torn header write during repair never compromises the existing quorum.
|
|
242
242
|
pub fn fuzz_quorum_repairs(
|
|
243
243
|
random: std.rand.Random,
|
|
244
244
|
comptime options: superblock_quorums.Options,
|
|
@@ -249,66 +249,66 @@ pub fn fuzz_quorum_repairs(
|
|
|
249
249
|
var q1: Quorums = undefined;
|
|
250
250
|
var q2: Quorums = undefined;
|
|
251
251
|
|
|
252
|
-
const
|
|
253
|
-
var
|
|
254
|
-
for (&
|
|
255
|
-
|
|
252
|
+
const headers_valid = blk: {
|
|
253
|
+
var headers: [superblock_copies]SuperBlockHeader = undefined;
|
|
254
|
+
for (&headers) |*header, i| {
|
|
255
|
+
header.* = std.mem.zeroInit(SuperBlockHeader, .{
|
|
256
256
|
.copy = @intCast(u8, i),
|
|
257
257
|
.version = SuperBlockVersion,
|
|
258
258
|
.replica = 1,
|
|
259
259
|
.storage_size_max = superblock.data_file_size_min,
|
|
260
260
|
.sequence = 123,
|
|
261
261
|
});
|
|
262
|
-
|
|
262
|
+
header.set_checksum();
|
|
263
263
|
}
|
|
264
|
-
break :blk
|
|
264
|
+
break :blk headers;
|
|
265
265
|
};
|
|
266
266
|
|
|
267
|
-
const
|
|
268
|
-
var
|
|
269
|
-
|
|
270
|
-
break :blk
|
|
267
|
+
const header_invalid = blk: {
|
|
268
|
+
var header = headers_valid[0];
|
|
269
|
+
header.checksum = 456;
|
|
270
|
+
break :blk header;
|
|
271
271
|
};
|
|
272
272
|
|
|
273
273
|
// Generate a random valid 2/4 quorum.
|
|
274
|
-
// 1 bits indicate valid
|
|
275
|
-
// 0 bits indicate invalid
|
|
274
|
+
// 1 bits indicate valid headers.
|
|
275
|
+
// 0 bits indicate invalid headers.
|
|
276
276
|
var valid = std.bit_set.IntegerBitSet(superblock_copies).initEmpty();
|
|
277
277
|
while (valid.count() < Quorums.Threshold.open.count() or random.boolean()) {
|
|
278
278
|
valid.set(random.uintLessThan(usize, superblock_copies));
|
|
279
279
|
}
|
|
280
280
|
|
|
281
|
-
var
|
|
282
|
-
for (&
|
|
283
|
-
|
|
281
|
+
var working_headers: [superblock_copies]SuperBlockHeader = undefined;
|
|
282
|
+
for (&working_headers) |*header, i| {
|
|
283
|
+
header.* = if (valid.isSet(i)) headers_valid[i] else header_invalid;
|
|
284
284
|
}
|
|
285
|
-
random.shuffle(
|
|
286
|
-
var
|
|
285
|
+
random.shuffle(SuperBlockHeader, &working_headers);
|
|
286
|
+
var repair_headers = working_headers;
|
|
287
287
|
|
|
288
|
-
const working_quorum = q1.working(&
|
|
288
|
+
const working_quorum = q1.working(&working_headers, .open) catch unreachable;
|
|
289
289
|
var quorum_repairs = working_quorum.repairs();
|
|
290
290
|
while (quorum_repairs.next()) |repair_copy| {
|
|
291
291
|
{
|
|
292
|
-
// Simulate a torn
|
|
293
|
-
var
|
|
294
|
-
|
|
295
|
-
const damaged_quorum = q2.working(&
|
|
296
|
-
assert(damaged_quorum.
|
|
292
|
+
// Simulate a torn header write, crash, recover sequence.
|
|
293
|
+
var damaged_headers = repair_headers;
|
|
294
|
+
damaged_headers[repair_copy] = header_invalid;
|
|
295
|
+
const damaged_quorum = q2.working(&damaged_headers, .open) catch unreachable;
|
|
296
|
+
assert(damaged_quorum.header.checksum == working_quorum.header.checksum);
|
|
297
297
|
}
|
|
298
298
|
|
|
299
299
|
// "Finish" the write so that we can test the next repair.
|
|
300
|
-
|
|
300
|
+
repair_headers[repair_copy] = headers_valid[repair_copy];
|
|
301
301
|
|
|
302
|
-
const quorum_repaired = q2.working(&
|
|
303
|
-
assert(quorum_repaired.
|
|
302
|
+
const quorum_repaired = q2.working(&repair_headers, .open) catch unreachable;
|
|
303
|
+
assert(quorum_repaired.header.checksum == working_quorum.header.checksum);
|
|
304
304
|
}
|
|
305
305
|
|
|
306
306
|
// At the end of all repairs, we expect to have every copy of the superblock.
|
|
307
307
|
// They do not need to be in their home slot.
|
|
308
308
|
var copies = Quorums.QuorumCount.initEmpty();
|
|
309
|
-
for (
|
|
310
|
-
assert(
|
|
311
|
-
copies.set(
|
|
309
|
+
for (repair_headers) |repair_header| {
|
|
310
|
+
assert(repair_header.checksum == working_quorum.header.checksum);
|
|
311
|
+
copies.set(repair_header.copy);
|
|
312
312
|
}
|
|
313
|
-
assert(
|
|
313
|
+
assert(repair_headers.len == copies.count());
|
|
314
314
|
}
|